Repository: cayleygraph/cayley Branch: master Commit: 81dcd7d73e45 Files: 378 Total size: 1.8 MB Directory structure: gitextract_q4kowbxs/ ├── .dockerignore ├── .gitbook.yaml ├── .github/ │ ├── ISSUE_TEMPLATE.md │ └── workflows/ │ └── build-and-release.yml ├── .gitignore ├── .goreleaser.yml ├── AUTHORS ├── CODEOWNERS ├── CONTRIBUTORS ├── Dockerfile ├── LICENSE ├── README.md ├── cayley_example.yml ├── client/ │ └── client.go ├── clog/ │ ├── clog.go │ └── glog/ │ └── glog.go ├── cmd/ │ ├── cayley/ │ │ ├── cayley.go │ │ └── command/ │ │ ├── convert.go │ │ ├── database.go │ │ ├── dedup.go │ │ ├── dump.go │ │ ├── health.go │ │ ├── http.go │ │ ├── repl.go │ │ └── schema.go │ ├── cayleyexport/ │ │ ├── cayleyexport.go │ │ └── cayleyexport_test.go │ ├── cayleyimport/ │ │ ├── cayleyimport.go │ │ └── cayleyimport_test.go │ ├── docgen/ │ │ └── docgen.go │ └── download_ui/ │ └── download_ui.go ├── configurations/ │ ├── default.json │ └── persisted.json ├── data/ │ ├── 30kmoviedata_gephi_meta.nq │ ├── people.jsonld │ ├── testdata.nq │ └── testdata_multigraph.nq ├── docs/ │ ├── 3rd-party-apis.md │ ├── GizmoAPI.md.in │ ├── README.md │ ├── SUMMARY.md │ ├── advanced-use.md │ ├── api/ │ │ └── swagger.yml │ ├── cayleyexport.md │ ├── cayleyimport.md │ ├── configuration.md │ ├── container.md │ ├── contributing.md │ ├── convert-linked-data-files.md │ ├── deployment/ │ │ ├── container.md │ │ └── k8s-1.md │ ├── docker-compose/ │ │ └── docker-compose.mongo.yml │ ├── faq.md │ ├── gephigraphstream.md │ ├── getting-involved/ │ │ ├── contributing.md │ │ ├── glossary.md │ │ ├── locations.md │ │ └── todo.md │ ├── getting-started.md │ ├── gizmoapi.md │ ├── glossary.md │ ├── graphql.md │ ├── gremlinapi.md │ ├── hacking.md │ ├── http.md │ ├── installation.md │ ├── k8s/ │ │ ├── README.md │ │ ├── cayley-mongo.yml │ │ ├── cayley-single.yml │ │ └── k8s.md │ ├── locations.md │ ├── migration.md │ ├── mql.md │ ├── query-languages/ │ │ ├── gephigraphstream.md │ │ ├── gizmoapi.md │ │ ├── graphql.md │ │ └── mql.md │ ├── quickstart-as-application.md │ ├── quickstart-as-lib.md │ ├── todo.md │ ├── tools/ │ │ └── convert-linked-data-files.md │ ├── ui-overview.md │ └── usage/ │ ├── 3rd-party-apis.md │ ├── advanced-use.md │ ├── http.md │ ├── migration.md │ ├── quickstart-as-lib.md │ └── ui-overview.md ├── examples/ │ ├── README.md │ ├── hello_bolt/ │ │ └── main.go │ ├── hello_schema/ │ │ └── main.go │ ├── hello_world/ │ │ └── main.go │ └── transaction/ │ └── main.go ├── go.mod ├── go.sum ├── gogen.go ├── graph/ │ ├── all/ │ │ ├── all.go │ │ └── all_cgo.go │ ├── gaedatastore/ │ │ ├── config.go │ │ ├── iterator.go │ │ ├── quadstore.go │ │ └── quadstore_test.go │ ├── graphmock/ │ │ └── graphmock.go │ ├── graphtest/ │ │ ├── graphtest.go │ │ ├── integration.go │ │ └── testutil/ │ │ └── testutil.go │ ├── hasa.go │ ├── hasa_test.go │ ├── http/ │ │ └── httpgraph.go │ ├── iterator/ │ │ ├── and.go │ │ ├── and_optimize.go │ │ ├── and_optimize_test.go │ │ ├── and_test.go │ │ ├── count.go │ │ ├── count_test.go │ │ ├── fixed.go │ │ ├── iterate.go │ │ ├── iterator.go │ │ ├── iterator_test.go │ │ ├── limit.go │ │ ├── limit_test.go │ │ ├── materialize.go │ │ ├── materialize_test.go │ │ ├── misc.go │ │ ├── not.go │ │ ├── not_test.go │ │ ├── or.go │ │ ├── or_test.go │ │ ├── recursive.go │ │ ├── recursive_test.go │ │ ├── regex.go │ │ ├── resolver.go │ │ ├── resolver_test.go │ │ ├── save.go │ │ ├── skip.go │ │ ├── skip_test.go │ │ ├── sort.go │ │ ├── unique.go │ │ ├── unique_test.go │ │ ├── value_comparison.go │ │ ├── value_comparison_test.go │ │ └── value_filter.go │ ├── kv/ │ │ ├── all/ │ │ │ └── all.go │ │ ├── all_iterator.go │ │ ├── badger/ │ │ │ ├── badger.go │ │ │ └── badger_test.go │ │ ├── bbolt/ │ │ │ ├── bolt.go │ │ │ └── bolt_test.go │ │ ├── bolt/ │ │ │ ├── bolt.go │ │ │ └── bolt_test.go │ │ ├── btree/ │ │ │ ├── btree.go │ │ │ └── btree_test.go │ │ ├── indexing.go │ │ ├── indexing_test.go │ │ ├── iterators.go │ │ ├── kvtest/ │ │ │ └── kvtest.go │ │ ├── leveldb/ │ │ │ ├── leveldb.go │ │ │ └── leveldb_test.go │ │ ├── metrics.go │ │ ├── quad_iterator.go │ │ ├── quadstore.go │ │ ├── quadstore_test.go │ │ └── registry.go │ ├── linksto.go │ ├── linksto_test.go │ ├── log/ │ │ └── graphlog.go │ ├── memstore/ │ │ ├── Makefile │ │ ├── all_iterator.go │ │ ├── gen.go │ │ ├── iterator.go │ │ ├── keys.go │ │ ├── keys_test.go │ │ ├── quadstore.go │ │ └── quadstore_test.go │ ├── nosql/ │ │ ├── all/ │ │ │ ├── all.go │ │ │ └── all_test.go │ │ ├── elastic/ │ │ │ └── elastic.go │ │ ├── iterator.go │ │ ├── mongo/ │ │ │ └── mongo.go │ │ ├── nosqltest/ │ │ │ └── nosqltest.go │ │ ├── ouch/ │ │ │ └── ouch.go │ │ ├── quadstore.go │ │ ├── shapes.go │ │ └── value_test.go │ ├── proto/ │ │ ├── primitive.pb.go │ │ ├── primitive.proto │ │ ├── primitive_helpers.go │ │ ├── serializations.pb.go │ │ ├── serializations.proto │ │ └── serializations_helpers.go │ ├── quadstore.go │ ├── quadwriter.go │ ├── quadwriter_test.go │ ├── refs/ │ │ └── refs.go │ ├── registry.go │ ├── sql/ │ │ ├── cockroach/ │ │ │ ├── cockroach.go │ │ │ └── cockroach_test.go │ │ ├── database.go │ │ ├── iterator.go │ │ ├── mysql/ │ │ │ ├── mysql.go │ │ │ └── mysql_test.go │ │ ├── optimizer.go │ │ ├── postgres/ │ │ │ ├── postgres.go │ │ │ └── postgres_test.go │ │ ├── quadstore.go │ │ ├── shape.go │ │ ├── shape_test.go │ │ ├── sqlite/ │ │ │ ├── sqlite.go │ │ │ └── sqlite_test.go │ │ └── sqltest/ │ │ └── sqltest.go │ ├── transaction.go │ └── transaction_test.go ├── imports.go ├── inference/ │ ├── inference.go │ └── inference_test.go ├── internal/ │ ├── decompressor/ │ │ ├── decompressor.go │ │ └── decompressor_test.go │ ├── dock/ │ │ └── dock.go │ ├── gephi/ │ │ ├── stream.go │ │ └── stream_test.go │ ├── http/ │ │ ├── api_v1.go │ │ ├── cors.go │ │ ├── health.go │ │ ├── http.go │ │ ├── http_test.go │ │ ├── logs.go │ │ ├── query.go │ │ └── write.go │ ├── linkedql/ │ │ └── schema/ │ │ ├── schema.go │ │ └── schema_test.go │ ├── load.go │ ├── lru/ │ │ ├── lru.go │ │ └── lru_test.go │ └── repl/ │ ├── repl.go │ └── repl_test.go ├── query/ │ ├── gizmo/ │ │ ├── environ.go │ │ ├── errors.go │ │ ├── finals.go │ │ ├── gizmo.go │ │ ├── gizmo_test.go │ │ └── traversals.go │ ├── graphql/ │ │ ├── graphql.go │ │ ├── graphql_test.go │ │ └── http.go │ ├── linkedql/ │ │ ├── entity_identfier.go │ │ ├── errors.go │ │ ├── graph_pattern.go │ │ ├── iter_docs.go │ │ ├── iter_tags.go │ │ ├── iter_tags_test.go │ │ ├── iter_values.go │ │ ├── jsonld_util.go │ │ ├── linkedql.go │ │ ├── property_path.go │ │ ├── registry.go │ │ ├── registry_test.go │ │ ├── step_types.go │ │ ├── steps/ │ │ │ ├── as.go │ │ │ ├── back.go │ │ │ ├── both.go │ │ │ ├── collect.go │ │ │ ├── count.go │ │ │ ├── difference.go │ │ │ ├── greater_than.go │ │ │ ├── greater_than_equals.go │ │ │ ├── has.go │ │ │ ├── has_reverse.go │ │ │ ├── in.go │ │ │ ├── intersect.go │ │ │ ├── jsonld_util.go │ │ │ ├── jsonld_util_test.go │ │ │ ├── labels.go │ │ │ ├── less_than.go │ │ │ ├── less_than_equals.go │ │ │ ├── like.go │ │ │ ├── limit.go │ │ │ ├── match.go │ │ │ ├── match_test.go │ │ │ ├── optional.go │ │ │ ├── order.go │ │ │ ├── out.go │ │ │ ├── placeholder.go │ │ │ ├── properties.go │ │ │ ├── property_names.go │ │ │ ├── property_names_as.go │ │ │ ├── regexp.go │ │ │ ├── reverse_properties.go │ │ │ ├── reverse_property_names.go │ │ │ ├── reverse_property_names_as.go │ │ │ ├── skip.go │ │ │ ├── steps_final.go │ │ │ ├── steps_test.go │ │ │ ├── test-cases/ │ │ │ │ ├── all-vertices.json │ │ │ │ ├── back.json │ │ │ │ ├── both.json │ │ │ │ ├── collect.json │ │ │ │ ├── count.json │ │ │ │ ├── difference.json │ │ │ │ ├── documents.json │ │ │ │ ├── greater-than-equals.json │ │ │ │ ├── greater-than.json │ │ │ │ ├── has-reverse.json │ │ │ │ ├── has.json │ │ │ │ ├── intersect.json │ │ │ │ ├── less-than-equals.json │ │ │ │ ├── less-than.json │ │ │ │ ├── like.json │ │ │ │ ├── limit.json │ │ │ │ ├── match-all.json │ │ │ │ ├── match-exact.json │ │ │ │ ├── optional.json │ │ │ │ ├── order.json │ │ │ │ ├── properties.json │ │ │ │ ├── property-names-as.json │ │ │ │ ├── property-names.json │ │ │ │ ├── reg-exp.json │ │ │ │ ├── reverse-properties.json │ │ │ │ ├── reverse-property-names-as.json │ │ │ │ ├── select-with-tags.json │ │ │ │ ├── select.json │ │ │ │ ├── skip.json │ │ │ │ ├── union.json │ │ │ │ ├── unique.json │ │ │ │ ├── view-reverse.json │ │ │ │ ├── view.json │ │ │ │ └── where.json │ │ │ ├── union.go │ │ │ ├── unique.go │ │ │ ├── vertex.go │ │ │ ├── visit.go │ │ │ ├── visit_reverse.go │ │ │ └── where.go │ │ └── voc_util.go │ ├── mql/ │ │ ├── build_iterator.go │ │ ├── fill.go │ │ ├── mql_test.go │ │ ├── query.go │ │ └── session.go │ ├── path/ │ │ ├── morphism_apply_functions.go │ │ ├── path.go │ │ ├── path_test.go │ │ └── pathtest/ │ │ └── pathtest.go │ ├── session.go │ ├── sexp/ │ │ ├── parser.go │ │ ├── parser_test.go │ │ └── session.go │ └── shape/ │ ├── path.go │ ├── shape.go │ └── shape_test.go ├── schema/ │ ├── loader.go │ ├── loader_test.go │ ├── namespaces.go │ ├── namespaces_test.go │ ├── schema.go │ ├── schema_test.go │ ├── types.go │ ├── writer.go │ └── writer_test.go ├── server/ │ └── http/ │ ├── accept.go │ ├── api_v2.go │ ├── api_v2_test.go │ └── common.go ├── ui/ │ ├── embed.go │ └── web/ │ ├── asset-manifest.json │ ├── gizmo.d.ts │ ├── index.html │ ├── manifest.json │ ├── precache-manifest.5316e0e4a35813e95b82d4799f1bb55c.js │ ├── robots.txt │ ├── service-worker.js │ └── static/ │ ├── css/ │ │ ├── 2.6b51f286.chunk.css │ │ └── main.72fe53e6.chunk.css │ └── js/ │ ├── 2.7d84b3fa.chunk.js │ ├── main.84d3ab8c.chunk.js │ └── runtime-main.0686c6e7.js ├── version/ │ └── version.go ├── vet.sh └── writer/ └── single.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ .idea/ vendor/ ================================================ FILE: .gitbook.yaml ================================================ root: ./docs/ ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ **Description** **Steps to reproduce the issue:** 1. 2. 3. **Received results:** ``` (program output or server response) ``` **Expected results:** **Output of `cayley version` or commit hash:** ``` (paste your output here) ``` **Environment details:** Backend database: `(database and version)` ================================================ FILE: .github/workflows/build-and-release.yml ================================================ name: Test and release on: push: branches: - master - ci_tests tags: - '*' pull_request: branches: - master env: DOCKER_IMAGE_NAME: ghcr.io/cayleygraph/cayley jobs: tests: name: Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version: '1.22' - name: Dependencies run: go mod download - name: Vet run: ./vet.sh - name: Build run: go build -v ./cmd/cayley - name: Test run: go test -v ./... release: name: Release runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/ci_tests') needs: - tests steps: - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version: '1.22' - name: Dependencies run: go mod download - name: Download UI run: go run cmd/download_ui/download_ui.go - name: Release uses: goreleaser/goreleaser-action@v6 if: startsWith(github.ref, 'refs/tags/v') with: version: latest args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} docker: name: Docker image runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/master') || (github.ref == 'refs/heads/ci_tests') needs: - tests steps: - uses: actions/checkout@v4 - name: Docker build run: | docker build -t $DOCKER_IMAGE_NAME:dev --build-arg VERSION=${{ github.ref_name }} . - name: Log in to the Container registry uses: docker/login-action@v3 if: startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/master') with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Push latest if: (github.ref == 'refs/heads/master') run: | docker push $DOCKER_IMAGE_NAME:dev - name: Push tagged if: startsWith(github.ref, 'refs/tags/') run: | docker tag $DOCKER_IMAGE_NAME:dev $DOCKER_IMAGE_NAME:${{ github.ref_name }} docker push $DOCKER_IMAGE_NAME:${{ github.ref_name }} - name: Push latest if: startsWith(github.ref, 'refs/tags/v') run: | docker tag $DOCKER_IMAGE_NAME:dev $DOCKER_IMAGE_NAME:latest docker push $DOCKER_IMAGE_NAME:latest ================================================ FILE: .gitignore ================================================ .idea/ db/ dist/ vendor/ *.swp main *.test *.peg.go cayley.cfg cayley.yml cayley.json .cayley_history .DS_Store packrd/packed-* # The build binary /cayley ================================================ FILE: .goreleaser.yml ================================================ builds: - main: ./cmd/cayley binary: cayley env: - CGO_ENABLED=0 goos: - linux - darwin - windows goarch: - amd64 - arm64 - arm ignore: - goos: windows goarch: arm - goos: windows goarch: arm64 - goos: darwin goarch: arm - goos: darwin goarch: arm64 ldflags: - -s -w -X github.com/cayleygraph/cayley/version.Version={{.Version}} -X github.com/cayleygraph/cayley/version.GitHash={{.FullCommit}} -X github.com/cayleygraph/cayley/version.BuildDate={{.Date}} archives: - format: tar.gz format_overrides: - goos: windows format: zip wrap_in_directory: true files: - README.md - LICENSE - AUTHORS - CONTRIBUTORS - configurations/* - data/* checksum: name_template: "checksums.txt" snapshot: name_template: "{{ .Tag }}-dev" changelog: sort: asc filters: exclude: - "^docs:" - "^test:" - "^ci:" snapcrafts: - name: cayley publish: false # TODO(dennwc): enable build when the package is reviewed summary: Open-source graph inspired by Freebase and Google's Knowledge Graph. description: | Cayley is an open-source graph inspired by the graph database behind Freebase and Google's Knowledge Graph. Its goal is to be a part of the developer's toolbox where Linked Data and graph-shaped data (semantic webs, social networks, etc) in general are concerned. grade: stable confinement: strict license: Apache-2.0 base: core18 apps: cayley: plugs: ["home", "network", "network-bind", "personal-files"] plugs: personal-files: read: - $HOME/.cayley write: - $HOME/.cayley ================================================ FILE: AUTHORS ================================================ # This is the official list of Cayley authors for copyright purposes. # This file is distinct from the CONTRIBUTORS files. # See the latter for an explanation. # # Names should be added to this file as: # Name or Organization # The email address is not required for organizations. # # Please keep the list sorted. Alexander Peters Andrew Dunham Bram Leenders Brendan Ball Denys Smirnov Derek Liang Iddan Aaronsohn Google Inc. Jay Graves Jeremy Jay Jørgen Teunis Pius Uzamere QlikTech International AB Robert Daniel Kortschak Robert Melton Stefan Koshiw Timothy Armstrong Yannic Bonenberger Zihua Li Gaurav Tiwari ================================================ FILE: CODEOWNERS ================================================ * @dennwc @eraserhd ================================================ FILE: CONTRIBUTORS ================================================ # The AUTHORS file lists the copyright holders; this file # lists people. For example, Google employees are listed here # but not in AUTHORS, because Google holds the copyright. # # Names should be added to this file as: # Name # # Please keep the list sorted. Alexander Peters Andrew Dunham Barak Michener Bram Leenders Brendan Ball Connor Newton Denys Smirnov Derek Liang Gaurav Tiwari Iddan Aaronsohn Jay Graves Jeremy Jay Jørgen Teunis Olaf Conradi Pius Uzamere Robert Daniel Kortschak Robert Melton Stefan Koshiw Tad Adams Timothy Armstrong Tyler Gibbons Yannic Bonenberger Zihua Li ================================================ FILE: Dockerfile ================================================ FROM golang:1.22 AS builder ARG VERSION=v0.8.x-dev # Create filesystem for minimal image WORKDIR /fs RUN mkdir -p etc/ssl/certs lib/x86_64-linux-gnu tmp bin data; \ # Copy CA Certificates cp /etc/ssl/certs/ca-certificates.crt etc/ssl/certs/ca-certificates.crt; \ # Copy C standard library cp /lib/x86_64-linux-gnu/libc.* lib/x86_64-linux-gnu/ # Set up workdir for compiling WORKDIR /src # Copy dependencies and install first COPY go.mod go.sum ./ RUN go mod download # Add all the other files ADD . . # Pass a Git short SHA as build information to be used for displaying version RUN GIT_SHA=$(git rev-parse --short=12 HEAD); \ go build \ -ldflags="-linkmode external -extldflags -static -X github.com/cayleygraph/cayley/version.Version=$VERSION -X github.com/cayleygraph/cayley/version.GitHash=$GIT_SHA" \ -a \ -installsuffix cgo \ -o /fs/bin/cayley \ -v \ ./cmd/cayley # Move persisted configuration into filesystem RUN mv configurations/persisted.json /fs/etc/cayley.json WORKDIR /fs # Initialize bolt indexes file RUN ./bin/cayley init --config etc/cayley.json FROM scratch # Copy filesystem as root COPY --from=builder /fs / # Define volume for configuration and data persistence. If you're using a # backend like bolt, make sure the file is saved to this directory. VOLUME [ "/data" ] EXPOSE 64210 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD [ "cayley", "health" ] # Adding everything to entrypoint allows us to init, load and serve only with # arguments passed to docker run. For example: # `docker run cayleygraph/cayley --init -i /data/my_data.nq` ENTRYPOINT ["cayley", "http", "--host=:64210"] ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. ================================================ FILE: README.md ================================================ ![Tests](https://github.com/cayleygraph/cayley/actions/workflows/build-and-release.yml/badge.svg) Cayley is an open-source database for [Linked Data](https://www.w3.org/standards/semanticweb/data). It is inspired by the graph database behind Google's [Knowledge Graph](https://en.wikipedia.org/wiki/Knowledge_Graph) (formerly [Freebase](https://en.wikipedia.org/wiki/Freebase_(database))). [![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-white.svg)](https://snapcraft.io/cayley) ## [Documentation](https://cayley.gitbook.io/cayley/) ## Features - Built-in query editor, visualizer and REPL - Multiple query languages: - [Gizmo](./docs/gizmoapi.md): query language inspired by [Gremlin](https://tinkerpop.apache.org/gremlin.html) - [GraphQL](./docs/graphql.md)-inspired query language - [MQL](./docs/mql.md): simplified version for [Freebase](https://en.wikipedia.org/wiki/Freebase_(database)) fans - Modular: easy to connect to your favorite programming languages and back-end stores - Production ready: well tested and used by various companies for their production workloads - Fast: optimized specifically for usage in applications ### Performance Rough performance testing shows that, on 2014 consumer hardware and an average disk, 134m quads in LevelDB is no problem and a multi-hop intersection query -- films starring X and Y -- takes ~150ms. ## Community - Website: [cayley.io](https://cayley.io) - Slack: [cayleygraph.slack.com](https://cayleygraph.slack.com) -- Invite [here](https://cayley-slackin.herokuapp.com/) - Discourse list: [discourse.cayley.io](https://discourse.cayley.io) (Also acts as mailing list, enable mailing list mode) - Twitter: [@cayleygraph](https://twitter.com/cayleygraph) - Involvement: [Contribute](./docs/contributing.md) ================================================ FILE: cayley_example.yml ================================================ store: # backend to use backend: bolt # address or path for the database address: "./cayley.db" # open database in read-only mode read_only: false # backend-specific options options: nosync: false query: timeout: 30s load: ignore_duplicates: false ignore_missing: false batch: 10000 ================================================ FILE: client/client.go ================================================ package client import ( "fmt" "io" "net/http" "net/url" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/pquads" ) func New(addr string) *Client { return &Client{addr: addr, cli: http.DefaultClient} } // Client is a struct used for communicating with a Cayley server through HTTP // Deprecated: Client exists for backwards compatability. New code should use // the updated client github.com/cayleygraph/go-client type Client struct { addr string cli *http.Client } func (c *Client) SetHTTPClient(cli *http.Client) { c.cli = cli } func (c *Client) url(s string, q map[string]string) string { addr := c.addr + s if len(q) != 0 { p := make(url.Values, len(q)) for k, v := range q { p.Set(k, v) } addr += "?" + p.Encode() } return addr } type errRequestFailed struct { Status string StatusCode int } func (e errRequestFailed) Error() string { return fmt.Sprintf("request failed: %d %v", e.StatusCode, e.Status) } func (c *Client) QuadReader() (quad.ReadCloser, error) { resp, err := http.Get(c.url("/api/v2/read", map[string]string{ "format": "pquads", })) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { resp.Body.Close() return nil, errRequestFailed{StatusCode: resp.StatusCode, Status: resp.Status} } r := pquads.NewReader(resp.Body, 10*1024*1024) r.SetCloser(resp.Body) return r, nil } type funcCloser struct { f func() error closed bool } func (c funcCloser) Close() error { if c.closed { return nil } c.closed = true return c.f() } func (c *Client) QuadWriter() (quad.WriteCloser, error) { pr, pw := io.Pipe() req, err := http.NewRequest("POST", c.url("/api/v2/write", nil), pr) if err != nil { return nil, err } req.Header.Set("Content-Type", pquads.ContentType) errc := make(chan error, 1) go func() { defer func() { close(errc) pr.Close() }() resp, err := c.cli.Do(req) if resp != nil && resp.Body != nil { defer resp.Body.Close() } if err == nil && resp.StatusCode != http.StatusOK { err = errRequestFailed{StatusCode: resp.StatusCode, Status: resp.Status} } errc <- err }() qw := pquads.NewWriter(pw, &pquads.Options{ Full: false, Strict: false, }) qw.SetCloser(funcCloser{f: func() error { pw.Close() return <-errc }}) return qw, nil } ================================================ FILE: clog/clog.go ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package clog provides a logging interface for cayley packages. package clog import "log" // Logger is the clog logging interface. type Logger interface { Infof(format string, args ...interface{}) Warningf(format string, args ...interface{}) Errorf(format string, args ...interface{}) Fatalf(format string, args ...interface{}) V(int) bool SetV(level int) } var logger Logger = &stdlog{ verbosity: 0, } // SetLogger set the clog logging implementation. func SetLogger(l Logger) { logger = l } var verbosity int // V returns whether the current clog verbosity is above the specified level. func V(level int) bool { if logger == nil { return false } return logger.V(level) } // SetV sets the clog verbosity level. func SetV(level int) { if logger != nil { logger.SetV(level) } } // Infof logs information level messages. func Infof(format string, args ...interface{}) { if logger != nil { logger.Infof(format, args...) } } // Warningf logs warning level messages. func Warningf(format string, args ...interface{}) { if logger != nil { logger.Warningf(format, args...) } } // Errorf logs error level messages. func Errorf(format string, args ...interface{}) { if logger != nil { logger.Errorf(format, args...) } } // Fatalf logs fatal messages and terminates the program. func Fatalf(format string, args ...interface{}) { if logger != nil { logger.Fatalf(format, args...) } } // stdlog wraps the standard library logger. type stdlog struct { verbosity int } func (stdlog) Infof(format string, args ...interface{}) { log.Printf(format, args...) } func (stdlog) Warningf(format string, args ...interface{}) { log.Printf("WARN: "+format, args...) } func (stdlog) Errorf(format string, args ...interface{}) { log.Printf("ERROR: "+format, args...) } func (stdlog) Fatalf(format string, args ...interface{}) { log.Fatalf("FATAL: "+format, args...) } func (s stdlog) V(level int) bool { return s.verbosity >= level } func (s *stdlog) SetV(level int) { s.verbosity = level } ================================================ FILE: clog/glog/glog.go ================================================ package glog import ( "fmt" "github.com/cayleygraph/cayley/clog" "github.com/golang/glog" ) func init() { clog.SetLogger(Logger{}) } type Logger struct{} func (Logger) Infof(format string, args ...interface{}) { glog.InfoDepth(3, fmt.Sprintf(format, args...)) } func (Logger) Warningf(format string, args ...interface{}) { glog.WarningDepth(3, fmt.Sprintf(format, args...)) } func (Logger) Errorf(format string, args ...interface{}) { glog.ErrorDepth(3, fmt.Sprintf(format, args...)) } func (Logger) Fatalf(format string, args ...interface{}) { glog.FatalDepth(3, fmt.Sprintf(format, args...)) } func (Logger) V(level int) bool { return bool(glog.V(glog.Level(level))) } func (Logger) SetV(v int) { glog.Warningf("changing log level is not supported; run command with '-v %d' flag", v) } ================================================ FILE: cmd/cayley/cayley.go ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build !appengine package main import ( "flag" "fmt" "net/http" _ "net/http/pprof" "os" "path/filepath" "strings" "github.com/cayleygraph/cayley/cmd/cayley/command" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/cayleygraph/cayley/clog" _ "github.com/cayleygraph/cayley/clog/glog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/version" "github.com/cayleygraph/quad" // Load supported backends _ "github.com/cayleygraph/cayley/graph/all" // Load all supported quad formats. _ "github.com/cayleygraph/quad/dot" _ "github.com/cayleygraph/quad/gml" _ "github.com/cayleygraph/quad/graphml" _ "github.com/cayleygraph/quad/json" _ "github.com/cayleygraph/quad/jsonld" _ "github.com/cayleygraph/quad/nquads" _ "github.com/cayleygraph/quad/pquads" // Load writer registry _ "github.com/cayleygraph/cayley/writer" // Load supported query languages _ "github.com/cayleygraph/cayley/query/gizmo" _ "github.com/cayleygraph/cayley/query/graphql" _ "github.com/cayleygraph/cayley/query/mql" _ "github.com/cayleygraph/cayley/query/sexp" ) var ( rootCmd = &cobra.Command{ Use: "cayley", Short: "Cayley is a graph store and graph query layer.", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { clog.Infof("Cayley version: %s (%s)", version.Version, version.GitHash) if conf, _ := cmd.Flags().GetString("config"); conf != "" { viper.SetConfigFile(conf) } err := viper.ReadInConfig() if _, ok := err.(viper.ConfigFileNotFoundError); !ok && err != nil { return err } if conf := viper.ConfigFileUsed(); conf != "" { wd, _ := os.Getwd() if rel, _ := filepath.Rel(wd, conf); rel != "" && strings.Count(rel, "..") < 3 { conf = rel } clog.Infof("using config file: %s", conf) } // force viper to load flags to variables graph.IgnoreDuplicates = viper.GetBool("load.ignore_duplicates") graph.IgnoreMissing = viper.GetBool("load.ignore_missing") quad.DefaultBatch = viper.GetInt("load.batch") if host, _ := cmd.Flags().GetString("pprof"); host != "" { go func() { if err := http.ListenAndServe(host, nil); err != nil { clog.Errorf("failed to run pprof handler: %v", err) } }() } if host, _ := cmd.Flags().GetString("metrics"); host != "" { go func() { if err := http.ListenAndServe(host, promhttp.Handler()); err != nil { clog.Errorf("failed to run metrics handler: %v", err) } }() } return nil }, } versionCmd = &cobra.Command{ Use: "version", Short: "Prints the version of Cayley.", // do not execute any persistent actions PersistentPreRun: func(cmd *cobra.Command, args []string) {}, Run: func(cmd *cobra.Command, args []string) { fmt.Println("Cayley version:", version.Version) fmt.Println("Git commit hash:", version.GitHash) if version.BuildDate != "" { fmt.Println("Build date:", version.BuildDate) } }, } ) type pFlag struct { flag.Value } func (pFlag) Type() string { return "string" } func init() { // set config names and paths viper.SetConfigName("cayley") viper.SetEnvPrefix("cayley") viper.AddConfigPath(".") viper.AddConfigPath("$HOME/.cayley/") viper.AddConfigPath("/etc/") if conf := os.Getenv("CAYLEY_CFG"); conf != "" { viper.SetConfigFile(conf) } rootCmd.AddCommand( versionCmd, command.NewInitDatabaseCmd(), command.NewLoadDatabaseCmd(), command.NewDumpDatabaseCmd(), command.NewUpgradeCmd(), command.NewReplCmd(), command.NewQueryCmd(), command.NewHTTPCmd(), command.NewConvertCmd(), command.NewDedupCommand(), command.NewHealthCmd(), command.NewSchemaCommand(), ) rootCmd.PersistentFlags().StringP("config", "c", "", "path to an explicit configuration file") qnames := graph.QuadStores() rootCmd.PersistentFlags().StringP("db", "d", "memstore", "database backend to use: "+strings.Join(qnames, ", ")) rootCmd.PersistentFlags().StringP("dbpath", "a", "", "path or address string for database") rootCmd.PersistentFlags().Bool("read_only", false, "open database in read-only mode") rootCmd.PersistentFlags().Bool("dup", true, "don't stop loading on duplicated on add") rootCmd.PersistentFlags().Bool("missing", false, "don't stop loading on missing key on delete") rootCmd.PersistentFlags().Int("batch", quad.DefaultBatch, "size of quads batch to load at once") rootCmd.PersistentFlags().String("memprofile", "", "path to output memory profile") rootCmd.PersistentFlags().String("cpuprofile", "", "path to output cpu profile") rootCmd.PersistentFlags().String("pprof", "", "host to serve pprof on (disabled by default)") rootCmd.PersistentFlags().String("metrics", "", "host to serve metrics on (disabled by default)") // bind flags to config variables viper.BindPFlag(command.KeyBackend, rootCmd.PersistentFlags().Lookup("db")) viper.BindPFlag(command.KeyAddress, rootCmd.PersistentFlags().Lookup("dbpath")) viper.BindPFlag(command.KeyReadOnly, rootCmd.PersistentFlags().Lookup("read_only")) viper.BindPFlag("load.ignore_duplicates", rootCmd.PersistentFlags().Lookup("dup")) viper.BindPFlag("load.ignore_missing", rootCmd.PersistentFlags().Lookup("missing")) viper.BindPFlag(command.KeyLoadBatch, rootCmd.PersistentFlags().Lookup("batch")) // make both store.path and store.address work viper.RegisterAlias(command.KeyPath, command.KeyAddress) // aliases for legacy config files viper.RegisterAlias("database", command.KeyBackend) viper.RegisterAlias("db_path", command.KeyAddress) viper.RegisterAlias("read_only", command.KeyReadOnly) viper.RegisterAlias("db_options", command.KeyOptions) { // re-register standard Go flags to cobra rf := rootCmd.PersistentFlags() flag.CommandLine.VisitAll(func(f *flag.Flag) { switch f.Name { case "v": // glog.v rf.VarP(pFlag{f.Value}, "verbose", "v", f.Usage) case "vmodule": // glog.vmodule rf.Var(pFlag{f.Value}, "vmodule", f.Usage) case "log_backtrace_at": // glog.log_backtrace_at rf.Var(pFlag{f.Value}, "backtrace", f.Usage) case "stderrthreshold": // glog.stderrthreshold rf.VarP(pFlag{f.Value}, "log", "l", f.Usage) case "alsologtostderr": // glog.alsologtostderr rf.Var(pFlag{f.Value}, f.Name, f.Usage) case "logtostderr": // glog.logtostderr f.Value.Set("true") rf.Var(pFlag{f.Value}, f.Name, f.Usage) case "log_dir": // glog.log_dir rf.Var(pFlag{f.Value}, "logs", f.Usage) } }) // make sure flags parsed flag is set - parse empty args flag.CommandLine = flag.NewFlagSet("", flag.ContinueOnError) flag.CommandLine.Parse([]string{""}) } } func main() { if err := rootCmd.Execute(); err != nil { clog.Errorf("%v", err) os.Exit(1) } } ================================================ FILE: cmd/cayley/command/convert.go ================================================ package command import ( "errors" "fmt" "io" "github.com/spf13/cobra" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/internal" "github.com/cayleygraph/quad" ) func newLazyReader(open func() (quad.ReadCloser, error)) quad.ReadCloser { return &lazyReader{open: open} } type lazyReader struct { rc quad.ReadCloser open func() (quad.ReadCloser, error) } func (r *lazyReader) ReadQuad() (quad.Quad, error) { if r.rc == nil { rc, err := r.open() if err != nil { return quad.Quad{}, err } r.rc = rc } return r.rc.ReadQuad() } func (r *lazyReader) Close() (err error) { if r.rc != nil { err = r.rc.Close() } return } type multiReader struct { rc []quad.ReadCloser i int } func (r *multiReader) ReadQuad() (quad.Quad, error) { for { if r.i >= len(r.rc) { return quad.Quad{}, io.EOF } rc := r.rc[r.i] q, err := rc.ReadQuad() if err == io.EOF { rc.Close() r.i++ continue } return q, err } } func (r *multiReader) Close() error { var first error if r.i < len(r.rc) { for _, rc := range r.rc[r.i:] { if err := rc.Close(); err != nil && first == nil { first = err } } } return nil } func NewConvertCmd() *cobra.Command { cmd := &cobra.Command{ Use: "convert", Aliases: []string{"conv"}, Short: "Convert quad files between supported formats.", RunE: func(cmd *cobra.Command, args []string) error { dump, _ := cmd.Flags().GetString(flagDump) dumpf, _ := cmd.Flags().GetString(flagDumpFormat) if dump == "" && len(args) > 0 { i := len(args) - 1 dump, args = args[i], args[:i] } var files []string if load, _ := cmd.Flags().GetString(flagLoad); load != "" { files = append(files, load) } files = append(files, args...) if len(files) == 0 || dump == "" { return errors.New("both input and output files must be specified") } loadf, _ := cmd.Flags().GetString(flagLoadFormat) var multi multiReader for _, path := range files { path := path multi.rc = append(multi.rc, newLazyReader(func() (quad.ReadCloser, error) { if dump == "-" { clog.Infof("reading %q", path) } else { fmt.Printf("reading %q\n", path) } return internal.QuadReaderFor(path, loadf) })) } // TODO: print additional stats return writerQuadsTo(dump, dumpf, &multi) }, } registerLoadFlags(cmd) registerDumpFlags(cmd) return cmd } ================================================ FILE: cmd/cayley/command/database.go ================================================ package command import ( "errors" "fmt" "os" "runtime" "runtime/pprof" "sort" "strings" "time" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/internal" "github.com/cayleygraph/quad" ) const ( KeyBackend = "store.backend" KeyAddress = "store.address" KeyPath = "store.path" KeyReadOnly = "store.read_only" KeyOptions = "store.options" KeyLoadBatch = "load.batch" ) const ( flagLoad = "load" flagLoadFormat = "load_format" flagDump = "dump" flagDumpFormat = "dump_format" ) var ErrNotPersistent = errors.New("database type is not persistent") func registerLoadFlags(cmd *cobra.Command) { // TODO: allow to load multiple files cmd.Flags().StringP(flagLoad, "i", "", `quad file to load after initialization (".gz" supported, "-" for stdin)`) var names []string for _, f := range quad.Formats() { if f.Reader != nil { names = append(names, f.Name) } } sort.Strings(names) cmd.Flags().String(flagLoadFormat, "", `quad file format to use for loading instead of auto-detection ("`+strings.Join(names, `", "`)+`")`) } func registerDumpFlags(cmd *cobra.Command) { cmd.Flags().StringP(flagDump, "o", "", `quad file to dump the database to (".gz" supported, "-" for stdout)`) var names []string for _, f := range quad.Formats() { if f.Writer != nil { names = append(names, f.Name) } } sort.Strings(names) cmd.Flags().String(flagDumpFormat, "", `quad file format to use instead of auto-detection ("`+strings.Join(names, `", "`)+`")`) } func NewInitDatabaseCmd() *cobra.Command { cmd := &cobra.Command{ Use: "init", Short: "Create an empty database.", RunE: func(cmd *cobra.Command, args []string) error { printBackendInfo() name := viper.GetString(KeyBackend) if graph.IsRegistered(name) && !graph.IsPersistent(name) { return ErrNotPersistent } // TODO: maybe check read-only flag in config before that? if err := initDatabase(); err != nil { return err } return nil }, } return cmd } func NewLoadDatabaseCmd() *cobra.Command { cmd := &cobra.Command{ Use: "load", Short: "Bulk-load a quad file into the database.", RunE: func(cmd *cobra.Command, args []string) error { printBackendInfo() p := mustSetupProfile(cmd) defer mustFinishProfile(p) load, _ := cmd.Flags().GetString(flagLoad) if load == "" && len(args) == 1 { load = args[0] } if load == "" { return errors.New("one quads file must be specified") } if init, err := cmd.Flags().GetBool("init"); err != nil { return err } else if init { if err = initDatabase(); err != nil { return err } } h, err := openDatabase() if err != nil { return err } defer h.Close() qw, err := h.NewQuadWriter() if err != nil { return err } defer qw.Close() // TODO: check read-only flag in config before that? typ, _ := cmd.Flags().GetString(flagLoadFormat) if err = internal.Load(qw, quad.DefaultBatch, load, typ); err != nil { return err } if dump, _ := cmd.Flags().GetString(flagDump); dump != "" { typ, _ := cmd.Flags().GetString(flagDumpFormat) if err = dumpDatabase(h, dump, typ); err != nil { return err } } return nil }, } cmd.Flags().Bool("init", false, "initialize the database before using it") registerLoadFlags(cmd) registerDumpFlags(cmd) return cmd } func NewDumpDatabaseCmd() *cobra.Command { cmd := &cobra.Command{ Use: "dump", Short: "Bulk-dump the database into a quad file.", RunE: func(cmd *cobra.Command, args []string) error { printBackendInfo() dump, _ := cmd.Flags().GetString(flagDump) if dump == "" && len(args) == 1 { dump = args[0] } if dump == "" { dump = "-" } h, err := openDatabase() if err != nil { return err } defer h.Close() typ, _ := cmd.Flags().GetString(flagDumpFormat) return dumpDatabase(h, dump, typ) }, } registerDumpFlags(cmd) return cmd } func NewUpgradeCmd() *cobra.Command { cmd := &cobra.Command{ Use: "upgrade", Short: "Upgrade Cayley database to current supported format.", RunE: func(cmd *cobra.Command, args []string) error { printBackendInfo() name := viper.GetString(KeyBackend) if graph.IsRegistered(name) && !graph.IsPersistent(name) { return ErrNotPersistent } addr := viper.GetString(KeyAddress) opts := graph.Options(viper.GetStringMap(KeyOptions)) clog.Infof("upgrading database...") return graph.UpgradeQuadStore(name, addr, opts) }, } return cmd } func printBackendInfo() { name := viper.GetString(KeyBackend) path := viper.GetString(KeyAddress) if path != "" { path = " (" + path + ")" } clog.Infof("using backend %q%s", name, path) } func initDatabase() error { name := viper.GetString(KeyBackend) path := viper.GetString(KeyAddress) opts := viper.GetStringMap(KeyOptions) return graph.InitQuadStore(name, path, graph.Options(opts)) } func openDatabase() (*graph.Handle, error) { name := viper.GetString(KeyBackend) path := viper.GetString(KeyAddress) opts := graph.Options(viper.GetStringMap(KeyOptions)) qs, err := graph.NewQuadStore(name, path, opts) if err != nil { return nil, err } qw, err := graph.NewQuadWriter("single", qs, opts) if err != nil { return nil, err } return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil } func openForQueries(cmd *cobra.Command) (*graph.Handle, error) { if init, err := cmd.Flags().GetBool("init"); err != nil { return nil, err } else if init { if err = initDatabase(); err == graph.ErrDatabaseExists { clog.Infof("database already initialized, skipping init") } else if err != nil { return nil, err } } var load string h, err := openDatabase() if err == graph.ErrQuadStoreNotPersistent { load = viper.GetString(KeyAddress) viper.Set(KeyAddress, "") h, err = openDatabase() } if err == graph.ErrQuadStoreNotPersistent { return nil, fmt.Errorf("%v; did you mean -i flag?", err) } else if err != nil { return nil, err } if load2, _ := cmd.Flags().GetString(flagLoad); load2 != "" { if load != "" { h.Close() return nil, fmt.Errorf("both -a and -i flags cannot be specified") } load = load2 } if load != "" { qw, err := h.NewQuadWriter() if err != nil { h.Close() return nil, err } defer qw.Close() typ, _ := cmd.Flags().GetString(flagLoadFormat) // TODO: check read-only flag in config before that? start := time.Now() if err = internal.Load(qw, quad.DefaultBatch, load, typ); err != nil { h.Close() return nil, err } clog.Infof("loaded %q in %v", load, time.Since(start)) } return h, nil } type profileData struct { cpuProfile *os.File memPath string } func mustSetupProfile(cmd *cobra.Command) profileData { p := profileData{} mpp := cmd.Flag("memprofile") p.memPath = mpp.Value.String() cpp := cmd.Flag("cpuprofile") v := cpp.Value.String() if v != "" { f, err := os.Create(v) if err != nil { fmt.Fprintf(os.Stderr, "Could not open CPU profile file %s\n", v) os.Exit(1) } p.cpuProfile = f pprof.StartCPUProfile(f) } return p } func mustFinishProfile(p profileData) { if p.cpuProfile != nil { pprof.StopCPUProfile() p.cpuProfile.Close() } if p.memPath != "" { f, err := os.Create(p.memPath) if err != nil { fmt.Fprintf(os.Stderr, "Could not open memory profile file %s\n", p.memPath) os.Exit(1) } runtime.GC() if err := pprof.WriteHeapProfile(f); err != nil { fmt.Fprintf(os.Stderr, "Could not write memory profile file %s\n", p.memPath) } f.Close() } } ================================================ FILE: cmd/cayley/command/dedup.go ================================================ package command import ( "context" "crypto/sha1" "errors" "fmt" "hash" "sort" "time" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc/rdf" ) func iriFlag(s string, err error) (quad.IRI, error) { if err != nil { return "", err } return quad.IRI(s), nil } func NewDedupCommand() *cobra.Command { cmd := &cobra.Command{ Use: "dedup", Short: "Deduplicate bnode values", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() printBackendInfo() h, err := openDatabase() if err != nil { return err } defer h.Close() pred, _ := iriFlag(cmd.Flags().GetString("pred")) typ, _ := iriFlag(cmd.Flags().GetString("type")) if typ == "" { return errors.New("no type is specified") } return dedupProperties(ctx, h, pred, typ) }, } cmd.Flags().String("pred", rdf.Type, "type predicate to use to find nodes") cmd.Flags().String("type", "", "type value to use to find nodes") return cmd } func valueLess(a, b graph.Ref) bool { // TODO(dennwc): more effective way s1, s2 := fmt.Sprint(a), fmt.Sprint(b) return s1 < s2 } type sortVals []graph.Ref func (a sortVals) Len() int { return len(a) } func (a sortVals) Less(i, j int) bool { return valueLess(a[i], a[j]) } func (a sortVals) Swap(i, j int) { a[i], a[j] = a[j], a[i] } type sortProp []property func (a sortProp) Len() int { return len(a) } func (a sortProp) Less(i, j int) bool { return valueLess(a[i].Pred, a[j].Pred) } func (a sortProp) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func hashProperties(h hash.Hash, m map[interface{}]property) string { props := make([]property, 0, len(m)) for _, p := range m { if len(p.Values) > 1 { sort.Sort(sortVals(p.Values)) } props = append(props, p) } sort.Sort(sortProp(props)) h.Reset() for _, p := range props { fmt.Fprint(h, p.Pred) h.Write([]byte{0}) for _, v := range p.Values { fmt.Fprint(h, v) h.Write([]byte{1}) } } res := make([]byte, 0, h.Size()) res = h.Sum(res) return string(res) } type property struct { Pred graph.Ref Values []graph.Ref } func dedupProperties(ctx context.Context, h *graph.Handle, pred, typ quad.IRI) error { batch := viper.GetInt(KeyLoadBatch) if batch == 0 { batch = quad.DefaultBatch } qs := h.QuadStore p := path.StartPath(qs).Has(pred, typ) ictx, cancel := context.WithCancel(ctx) defer cancel() var gerr error seen := make(map[string]graph.Ref) cnt, dedup := 0, 0 start := time.Now() last := start hh := sha1.New() tx := graph.NewTransaction() txn := 0 flush := func() { if txn == 0 { return } err := h.ApplyTransaction(tx) if err == nil { tx = graph.NewTransaction() dedup += txn txn = 0 } else { gerr = err cancel() } if now := time.Now(); now.Sub(last) > time.Second*5 { last = now clog.Infof("deduplicated %d/%d nodes (%.1f nodes/sec)", dedup, cnt, float64(cnt)/now.Sub(start).Seconds(), ) } } err := p.Iterate(ictx).Each(func(s graph.Ref) error { cnt++ it := qs.QuadIterator(quad.Subject, s).Iterate() defer it.Close() m := make(map[interface{}]property) for it.Next(ictx) { q := it.Result() p, err := qs.QuadDirection(q, quad.Predicate) if err != nil { return err } o, err := qs.QuadDirection(q, quad.Object) if err != nil { return err } k := refs.ToKey(p) prop := m[k] prop.Pred = p prop.Values = append(prop.Values, o) m[k] = prop } if gerr = it.Err(); gerr != nil { cancel() } ph := hashProperties(hh, m) id, ok := seen[ph] if !ok { seen[ph] = s return nil } if gerr = dedupValueTx(ictx, h, tx, s, id); gerr != nil { cancel() } txn++ if txn >= batch { // TODO(dennwc): flag flush() } return nil }) flush() clog.Infof("deduplicated %d/%d nodes in %v", dedup, cnt, time.Since(start)) if gerr != nil { err = gerr } return err } func dedupValueTx(ctx context.Context, h *graph.Handle, tx *graph.Transaction, a, b graph.Ref) error { v, err := h.NameOf(b) if err != nil { return err } it := h.QuadIterator(quad.Object, a).Iterate() defer it.Close() for it.Next(ctx) { // TODO(dennwc): we should be able to add "raw" quads without getting values for directions q, err := h.Quad(it.Result()) if err != nil { return err } tx.RemoveQuad(q) q.Object = v tx.AddQuad(q) } if err := it.Err(); err != nil { return err } it.Close() it = h.QuadIterator(quad.Subject, a).Iterate() defer it.Close() for it.Next(ctx) { q, err := h.Quad(it.Result()) if err != nil { return err } tx.RemoveQuad(q) } if err := it.Err(); err != nil { return err } return nil } ================================================ FILE: cmd/cayley/command/dump.go ================================================ package command import ( "compress/gzip" "fmt" "io" "os" "path/filepath" "strings" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/quad" ) func writerQuadsTo(path string, typ string, qr quad.Reader) error { var f *os.File if path == "-" { f = os.Stdout clog.Infof("writing quads to stdout") } else { var err error f, err = os.Create(path) if err != nil { return fmt.Errorf("could not create file %q: %v", path, err) } defer f.Close() fmt.Printf("writing quads to file %q\n", path) } var w io.Writer = f ext := filepath.Ext(path) if ext == ".gz" { ext = filepath.Ext(strings.TrimSuffix(path, ext)) gzip := gzip.NewWriter(f) defer gzip.Close() w = gzip } var format *quad.Format if typ == "" { format = quad.FormatByExt(ext) if format == nil { typ = "nquads" } } if format == nil { format = quad.FormatByName(typ) } if format == nil { return fmt.Errorf("unsupported format: %q", typ) } else if format.Writer == nil { return fmt.Errorf("encoding in %s format is not supported", typ) } qw := format.Writer(w) defer qw.Close() n, err := quad.Copy(qw, qr) if err != nil { return err } else if err = qw.Close(); err != nil { return err } if path != "-" { fmt.Printf("%d entries were written\n", n) } return nil } func dumpDatabase(h *graph.Handle, path string, typ string) error { //TODO: add possible support for exporting specific queries only qr := graph.NewQuadStoreReader(h.QuadStore) defer qr.Close() return writerQuadsTo(path, typ, qr) } ================================================ FILE: cmd/cayley/command/health.go ================================================ package command import ( "fmt" "log" "net/http" "github.com/spf13/cobra" ) const defaultAddress = "http://localhost:64210/" func NewHealthCmd() *cobra.Command { return &cobra.Command{ Use: "health", Aliases: []string{}, Short: "Health check HTTP server", RunE: func(cmd *cobra.Command, args []string) error { if len(args) > 1 { return fmt.Errorf("Too many arguments provided, expected 0 or 1") } address := defaultAddress if len(args) == 1 { address = args[0] } healthAddress := address + "health" resp, err := http.Get(healthAddress) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != 204 { return fmt.Errorf("/health responded with status code %d, expected 204", resp.StatusCode) } log.Printf("%s ok", healthAddress) return nil }, } } ================================================ FILE: cmd/cayley/command/http.go ================================================ package command import ( "net" "net/http" "time" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/cayleygraph/cayley/clog" chttp "github.com/cayleygraph/cayley/internal/http" ) func NewHTTPCmd() *cobra.Command { cmd := &cobra.Command{ Use: "http", Short: "Serve an HTTP endpoint on the given host and port.", RunE: func(cmd *cobra.Command, args []string) error { printBackendInfo() p := mustSetupProfile(cmd) defer mustFinishProfile(p) h, err := openForQueries(cmd) if err != nil { return err } defer h.Close() err = chttp.SetupRoutes(h, &chttp.Config{ Timeout: viper.GetDuration(keyQueryTimeout), ReadOnly: viper.GetBool(KeyReadOnly), }) if err != nil { return err } host, _ := cmd.Flags().GetString("host") phost := host if host, port, err := net.SplitHostPort(host); err == nil && host == "" { phost = net.JoinHostPort("localhost", port) } clog.Infof("listening on %s, web interface at http://%s", host, phost) return http.ListenAndServe(host, nil) }, } cmd.Flags().String("host", "127.0.0.1:64210", "host:port to listen on") cmd.Flags().Bool("init", false, "initialize the database before using it") cmd.Flags().DurationP("timeout", "t", 30*time.Second, "elapsed time until an individual query times out") registerLoadFlags(cmd) viper.BindPFlag(keyQueryTimeout, cmd.Flags().Lookup("timeout")) return cmd } ================================================ FILE: cmd/cayley/command/repl.go ================================================ package command import ( "context" "encoding/json" "fmt" "io/ioutil" "os" "os/signal" "strings" "time" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/internal/repl" "github.com/cayleygraph/cayley/query" ) const ( keyQueryTimeout = "query.timeout" ) func getContext() (context.Context, func()) { ctx, cancel := context.WithCancel(context.Background()) ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt) go func() { select { case <-ch: case <-ctx.Done(): } signal.Stop(ch) cancel() }() return ctx, cancel } func registerQueryFlags(cmd *cobra.Command) { langs := query.Languages() cmd.Flags().Bool("init", false, "initialize the database before using it") cmd.Flags().String("lang", "gizmo", `query language to use ("`+strings.Join(langs, `", "`)+`")`) cmd.Flags().DurationP("timeout", "t", 30*time.Second, "elapsed time until an individual query times out") viper.BindPFlag(keyQueryTimeout, cmd.Flags().Lookup("timeout")) registerLoadFlags(cmd) } func NewReplCmd() *cobra.Command { cmd := &cobra.Command{ Use: "repl", Short: "Drop into a REPL of the given query language.", RunE: func(cmd *cobra.Command, args []string) error { printBackendInfo() p := mustSetupProfile(cmd) defer mustFinishProfile(p) h, err := openForQueries(cmd) if err != nil { return err } defer h.Close() ctx, cancel := getContext() defer cancel() timeout := viper.GetDuration("timeout") lang, _ := cmd.Flags().GetString("lang") return repl.Repl(ctx, h, lang, timeout) }, } registerQueryFlags(cmd) return cmd } func NewQueryCmd() *cobra.Command { cmd := &cobra.Command{ Use: "query", Aliases: []string{"qu"}, Short: "Run a query in a specified database and print results.", RunE: func(cmd *cobra.Command, args []string) error { var querystr string if len(args) == 0 { bytes, err := ioutil.ReadAll(os.Stdin) if err != nil { return fmt.Errorf("error occured while reading from stdin : %s", err) } querystr = string(bytes) } else if len(args) == 1 { querystr = args[0] } else { return fmt.Errorf("query accepts only one argument, the query string or nothing for reading from stdin") } clog.Infof("Query:\n%s", querystr) printBackendInfo() p := mustSetupProfile(cmd) defer mustFinishProfile(p) h, err := openForQueries(cmd) if err != nil { return err } defer h.Close() ctx, cancel := getContext() defer cancel() timeout := viper.GetDuration("timeout") if timeout > 0 { ctx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } lang, _ := cmd.Flags().GetString("lang") limit, err := cmd.Flags().GetInt("limit") if err != nil { return err } enc := json.NewEncoder(os.Stdout) it, err := query.Execute(ctx, h, lang, querystr, query.Options{ Collation: query.JSON, Limit: limit, }) if err != nil { return err } defer it.Close() for i := 0; it.Next(ctx) && (limit <= 0 || i < limit); i++ { if err = enc.Encode(it.Result()); err != nil { return err } } return it.Err() }, } registerQueryFlags(cmd) cmd.Flags().IntP("limit", "n", 100, "limit a number of results") return cmd } ================================================ FILE: cmd/cayley/command/schema.go ================================================ package command import ( "bytes" "encoding/json" "fmt" "github.com/spf13/cobra" "github.com/cayleygraph/cayley/internal/linkedql/schema" ) func NewSchemaCommand() *cobra.Command { root := &cobra.Command{ Use: "schema", Short: "Commands related to RDF schema", } root.AddCommand( NewLinkedQLSchemaCommand(), ) return root } func NewLinkedQLSchemaCommand() *cobra.Command { return &cobra.Command{ Use: "linkedql", Short: "Generate LinkedQL Schema to stdout", RunE: func(cmd *cobra.Command, args []string) error { if len(args) > 0 { return fmt.Errorf("too many arguments provided, expected 0") } data := schema.Generate() buf := bytes.NewBuffer(nil) err := json.Indent(buf, data, "", "\t") if err != nil { return err } fmt.Println(buf) return nil }, } } ================================================ FILE: cmd/cayleyexport/cayleyexport.go ================================================ package main import ( "io" "net/http" "os" "path/filepath" "github.com/cayleygraph/cayley/clog" // Load all supported quad formats. "github.com/cayleygraph/quad" _ "github.com/cayleygraph/quad/jsonld" _ "github.com/cayleygraph/quad/nquads" "github.com/spf13/cobra" ) const defaultFormat = "jsonld" // NewCmd creates the command func NewCmd() *cobra.Command { var quiet bool var uri, formatName, out string var cmd = &cobra.Command{ Use: "cayleyexport", Short: "Export data from Cayley. If no file is provided, cayleyexport writes to stdout.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { if quiet { clog.SetV(500) } var format *quad.Format var w io.Writer if formatName != "" { format = quad.FormatByName(formatName) } if out == "" { w = cmd.OutOrStdout() } else { if formatName == "" { format = formatByFileName(out) if format == nil { clog.Warningf("File has unknown extension %v. Defaulting to %v", out, defaultFormat) } } file, err := os.Create(out) if err != nil { return err } w = file defer file.Close() } if format == nil { format = quad.FormatByName(defaultFormat) } req, err := http.NewRequest(http.MethodGet, uri+"/api/v2/read", nil) req.Header.Set("Accept", format.Mime[0]) if err != nil { return err } client := &http.Client{} resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() _, err = io.Copy(w, resp.Body) if err != nil { return err } return nil }, } cmd.Flags().StringVarP(&uri, "uri", "", "http://127.0.0.1:64210", "Cayley URI connection string") cmd.Flags().StringVarP(&formatName, "format", "", "", "format of the provided data (if can not be detected defaults to JSON-LD)") cmd.Flags().StringVarP(&out, "out", "o", "", "output file; if not specified, stdout is used") cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "hide all log output") return cmd } func main() { cmd := NewCmd() if err := cmd.Execute(); err != nil { os.Exit(1) } } func formatByFileName(fileName string) *quad.Format { ext := filepath.Ext(fileName) return quad.FormatByExt(ext) } ================================================ FILE: cmd/cayleyexport/cayleyexport_test.go ================================================ package main import ( "bytes" "fmt" "net" "net/http" "testing" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/jsonld" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/memstore" chttp "github.com/cayleygraph/cayley/internal/http" ) var testData = []quad.Quad{ { Subject: quad.IRI("http://example.com/alice"), Predicate: quad.IRI("http://example.com/likes"), Object: quad.IRI("http://example.com/bob"), Label: nil, }, } func serializeTestData() string { buf := bytes.NewBuffer(nil) w := jsonld.NewWriter(buf) w.WriteQuads(testData) w.Close() return buf.String() } func TestCayleyExport(t *testing.T) { qs := memstore.New(testData...) qw, err := graph.NewQuadWriter("single", qs, graph.Options{}) require.NoError(t, err) h := &graph.Handle{QuadStore: qs, QuadWriter: qw} chttp.SetupRoutes(h, &chttp.Config{}) lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { lis.Close() }) srv := &http.Server{ Addr: lis.Addr().String(), } go srv.Serve(lis) cmd := NewCmd() b := bytes.NewBufferString("") cmd.SetOut(b) cmd.SetArgs([]string{ "--uri", fmt.Sprintf("http://%s", lis.Addr().String()), }) err = cmd.Execute() require.NoError(t, err) data := serializeTestData() require.NotEmpty(t, data) require.Equal(t, data, b.String()) } ================================================ FILE: cmd/cayleyimport/cayleyimport.go ================================================ package main import ( "encoding/json" "errors" "fmt" "io" "io/ioutil" "net/http" "os" "path/filepath" "github.com/cayleygraph/cayley/clog" // Load all supported quad formats. "github.com/cayleygraph/quad" _ "github.com/cayleygraph/quad/jsonld" _ "github.com/cayleygraph/quad/nquads" "github.com/spf13/cobra" ) const defaultFormat = "jsonld" // NewCmd creates the command func NewCmd() *cobra.Command { var quiet bool var uri, formatName string var cmd = &cobra.Command{ Use: "cayleyimport ", Short: "Import data into Cayley. If no file is provided, cayleyimport reads from stdin.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if quiet { clog.SetV(500) } var format *quad.Format var reader io.Reader if formatName != "" { format = quad.FormatByName(formatName) } if len(args) == 0 { in := cmd.InOrStdin() if !hasIn(in) { return errors.New("Either provide file to read from or pipe data") } reader = in } else { fileName := args[0] if formatName == "" { format = formatByFileName(fileName) if format == nil { clog.Warningf("File has unknown extension %v. Defaulting to %v", fileName, defaultFormat) } } file, err := os.Open(fileName) if err != nil { return err } defer file.Close() reader = file } if format == nil { format = quad.FormatByName(defaultFormat) } r, err := http.Post(uri+"/api/v2/write", format.Mime[0], reader) if err != nil { return err } defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) if err != nil { return err } if r.StatusCode == http.StatusOK { var response struct { Result string `json:"result"` Count string `json:"count"` Error string `json:"error"` } json.Unmarshal(body, &response) if response.Error != "" { return errors.New(response.Error) } if !quiet { fmt.Println(response.Result) } } else if r.StatusCode == http.StatusNotFound { return errors.New("Database instance does not support write") } return nil }, } cmd.Flags().StringVarP(&uri, "uri", "", "http://127.0.0.1:64210", "Cayley URI connection string") cmd.Flags().StringVarP(&formatName, "format", "", "", "format of the provided data (if can not be detected defaults to JSON-LD)") cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "hide all log output") return cmd } func main() { cmd := NewCmd() if err := cmd.Execute(); err != nil { os.Exit(1) } } func hasIn(in io.Reader) bool { if in == os.Stdin { stat, _ := os.Stdin.Stat() return (stat.Mode() & os.ModeCharDevice) == 0 } return true } func formatByFileName(fileName string) *quad.Format { ext := filepath.Ext(fileName) return quad.FormatByExt(ext) } ================================================ FILE: cmd/cayleyimport/cayleyimport_test.go ================================================ package main import ( "bytes" "context" "fmt" "net" "net/http" "path" "sort" "testing" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/memstore" chttp "github.com/cayleygraph/cayley/internal/http" ) var expectData = []quad.Quad{ {quad.IRI("http://example.com/alice"), quad.IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), quad.IRI("http://xmlns.com/foaf/0.1/Person"), quad.Value(nil)}, {quad.IRI("http://example.com/alice"), quad.IRI("http://xmlns.com/foaf/0.1/knows"), quad.IRI("http://example.com/bob"), nil}, {quad.IRI("http://example.com/alice"), quad.IRI("http://xmlns.com/foaf/0.1/name"), quad.String("Alice"), nil}, {quad.IRI("http://example.com/bob"), quad.IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), quad.IRI("http://xmlns.com/foaf/0.1/Person"), nil}, {quad.IRI("http://example.com/bob"), quad.IRI("http://xmlns.com/foaf/0.1/knows"), quad.IRI("http://example.com/alice"), nil}, {quad.IRI("http://example.com/bob"), quad.IRI("http://xmlns.com/foaf/0.1/name"), quad.String("Bob"), nil}, } func allQuads(t testing.TB, qs graph.QuadStore) []quad.Quad { ctx := context.Background() it := qs.QuadsAllIterator().Iterate() defer it.Close() var out []quad.Quad for it.Next(ctx) { ref := it.Result() q, err := qs.Quad(ref) require.NoError(t, err) out = append(out, q) } require.NoError(t, it.Err()) sort.Sort(quad.ByQuadString(out)) return out } func TestCayleyImport(t *testing.T) { qs := memstore.New() qw, err := graph.NewQuadWriter("single", qs, graph.Options{}) require.NoError(t, err) h := &graph.Handle{QuadStore: qs, QuadWriter: qw} chttp.SetupRoutes(h, &chttp.Config{}) lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { lis.Close() }) srv := &http.Server{ Addr: lis.Addr().String(), } go srv.Serve(lis) cmd := NewCmd() b := bytes.NewBufferString("") cmd.SetOut(b) fileName := path.Join("..", "..", "data", "people.jsonld") cmd.SetArgs([]string{ fileName, "--uri", fmt.Sprintf("http://%s", lis.Addr().String()), }) err = cmd.Execute() require.NoError(t, err) require.Empty(t, b.String()) sort.Sort(quad.ByQuadString(expectData)) require.Equal(t, expectData, allQuads(t, qs)) } ================================================ FILE: cmd/docgen/docgen.go ================================================ package main import ( "bufio" "bytes" "flag" "fmt" "go/ast" "go/doc" "go/parser" "go/token" "io" "os" "path/filepath" "regexp" "strings" ) var ( packageName = flag.String("pck", "github.com/cayleygraph/cayley/query/gizmo", "") out = flag.String("o", "-", "output file") in = flag.String("i", "", "input file") ) const placeholder = `#AUTOGENERATED#` func main() { flag.Parse() fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, "./query/gizmo", nil, parser.ParseComments) if err != nil { panic(err) } p := pkgs[filepath.Base(*packageName)] dp := doc.New(p, *packageName, doc.AllDecls) var w io.Writer = os.Stdout if fname := *out; fname != "" && fname != "-" { f, err := os.Create(fname) if err != nil { panic(err) } defer f.Close() w = f } var r io.Reader = strings.NewReader(placeholder) if fname := *in; fname != "" { f, err := os.Open(fname) if err != nil { panic(err) } defer f.Close() r = f } sc := bufio.NewScanner(r) for sc.Scan() { line := bytes.TrimSpace(sc.Bytes()) if bytes.Equal(line, []byte(placeholder)) { writeDocs(w, dp) } else { w.Write(line) w.Write([]byte("\n")) } } } func writeDocs(w io.Writer, dp *doc.Package) { type Type struct { Title string Name string } names := map[string]Type{ "graphObject": { Title: "The `graph` object", Name: "graph", }, "pathObject": { Title: "Path object", Name: "path", }, } for _, tp := range dp.Types { t, ok := names[tp.Name] if !ok { continue } s := tp.Doc if i := strings.IndexAny(s, "\n\r"); i >= 0 { s = s[i+1:] } s = strings.TrimSpace(s) fmt.Fprintf(w, "## %s\n\n", t.Title) fmt.Fprintf(w, "%s\n\n", funcDocs(s)) for _, m := range tp.Methods { if !isExported(m.Name) { continue } m.Doc = strings.TrimSpace(m.Doc) sig := Signature(m) fmt.Fprintf(w, "### `%s.%s%s`\n\n%s\n\n", t.Name, m.Name, sig, funcDocs(m.Doc)) } } } var reSignature = regexp.MustCompile(`Signature:\s+\((.+)\)`) func Signature(m *doc.Func) string { if reSignature.MatchString(m.Doc) { sub := reSignature.FindStringSubmatch(m.Doc) m.Doc = strings.Replace(m.Doc, sub[0], "", 1) return "(" + sub[1] + ")" } tp := m.Decl.Type if isJsArgs(tp.Params) { return "(*)" } var names []string for _, a := range tp.Params.List { for _, name := range a.Names { names = append(names, name.Name) } } buf := bytes.NewBuffer(nil) buf.WriteRune('(') buf.WriteString(strings.Join(names, ", ")) buf.WriteRune(')') return buf.String() } func isExported(s string) bool { return ast.IsExported(s) } func isJsArgs(f *ast.FieldList) bool { if len(f.List) != 1 { return false } p := f.List[0] if len(p.Names) != 1 { return false } sel, ok := p.Type.(*ast.SelectorExpr) if !ok { return false } return sel.Sel.Name == "FunctionCall" } var reScript = regexp.MustCompile(`//\s*(\w+)`) func funcDocs(s string) string { if s == "" { return "TODO: docs" } buf := bytes.NewBuffer(nil) buf.Grow(len(s)) sc := bufio.NewScanner(strings.NewReader(s)) const defaultLang = "" var ( inCode bool lang string ) for sc.Scan() { line := sc.Text() if code := strings.HasPrefix(line, "\t"); code { if !inCode { inCode = true lang = defaultLang skip := false if reScript.MatchString(line) { skip = true lang = reScript.FindStringSubmatch(line)[1] } buf.WriteString("```") buf.WriteString(lang) buf.WriteString("\n") if skip { continue } } line = strings.TrimPrefix(line, "\t") } else if inCode && !code { inCode = false buf.WriteString("```\n") } buf.WriteString(line) buf.WriteRune('\n') } if inCode { buf.WriteString("```\n") } return buf.String() } ================================================ FILE: cmd/download_ui/download_ui.go ================================================ package main import ( "archive/zip" "fmt" "io" "log" "net/http" "os" "path/filepath" "strings" ) const ( version = "v0.8.0" fileURL = "https://github.com/cayleygraph/web/releases/download/" + version + "/web.zip" fileName = "web.zip" directoryName = "ui/web" ) func main() { log.Printf("Downloading %s to %s...", fileURL, fileName) if err := DownloadFile(fileName, fileURL); err != nil { panic(err) } log.Printf("Downloaded %s to %s", fileURL, fileName) log.Printf("Extracting %s to %s...", fileName, directoryName) err := Unzip(fileName, directoryName) if err != nil { panic(err) } log.Printf("Extracted %s to %s/", fileName, directoryName) err = os.Remove(fileName) if err != nil { panic(err) } log.Printf("Removed %s", fileName) } // DownloadFile will download a url to a local file. It's efficient because it will // write as it downloads and not load the whole file into memory. func DownloadFile(filepath string, url string) error { // Get the data resp, err := http.Get(url) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != 200 { return fmt.Errorf("Received %v status code instead of 200 for %v", resp.Status, url) } // Create the file out, err := os.Create(filepath) if err != nil { return err } defer out.Close() // Write the body to file _, err = io.Copy(out, resp.Body) return err } // Unzip will decompress a zip archive, moving all files and folders // within the zip file (parameter 1) to an output directory (parameter 2). func Unzip(src string, dest string) error { r, err := zip.OpenReader(src) if err != nil { return err } defer r.Close() for _, f := range r.File { // Store filename/path for returning and using later on fpath := filepath.Join(dest, f.Name) // Check for ZipSlip. More Info: http://bit.ly/2MsjAWE if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { return fmt.Errorf("%s: illegal file path", fpath) } if f.FileInfo().IsDir() { // Make Folder os.MkdirAll(fpath, 0755) continue } // Make File if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { return err } outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return err } rc, err := f.Open() if err != nil { return err } _, err = io.Copy(outFile, rc) if err != nil { return err } // Close the file without defer to close before next iteration of loop err = outFile.Close() if err != nil { return err } err = rc.Close() if err != nil { return err } } return nil } ================================================ FILE: configurations/default.json ================================================ { "store": { "backend": "memstore" } } ================================================ FILE: configurations/persisted.json ================================================ { "store": { "backend": "bolt", "address": "data/cayley.db" } } ================================================ FILE: data/30kmoviedata_gephi_meta.nq ================================================ "true"^^ . "true"^^ . "true"^^ . ================================================ FILE: data/people.jsonld ================================================ { "@context": { "ex": "http://example.com/", "@vocab": "http://xmlns.com/foaf/0.1/" }, "@graph": [ { "@id": "ex:alice", "@type": "Person", "name": "Alice", "knows": { "@id": "ex:bob" } }, { "@id": "ex:bob", "@type": "Person", "name": "Bob", "knows": { "@id": "ex:alice" } } ] } ================================================ FILE: data/testdata.nq ================================================ . . "cool_person" . . . . . "cool_person" . . . "cool_person" . . . "smart_person" . "smart_person" . ================================================ FILE: data/testdata_multigraph.nq ================================================ . . "cool_person" . . . . . "cool_person" . . . "cool_person" . . . "smart_person" . "smart_person" . "smart_person" . ================================================ FILE: docs/3rd-party-apis.md ================================================ # 3rd-Party-APIs Various 3rd party APIs * **Clojure**: [https://github.com/wgb/cayley-clj](https://github.com/wgb/cayley-clj) * **Javascript/NodeJS**: [https://github.com/lnshi/node-cayley](https://github.com/lnshi/node-cayley), [https://github.com/villadora/cayley.js](https://github.com/villadora/cayley.js) * **Ruby**: [https://github.com/reneklacan/cayley-ruby](https://github.com/reneklacan/cayley-ruby) * **PHP**: [https://github.com/mcuadros/php-cayley](https://github.com/mcuadros/php-cayley) * **Python**: [https://github.com/ziyasal/pyley](https://github.com/ziyasal/pyley) * **.NET**: [https://github.com/ziyasal/Cayley.Net](https://github.com/ziyasal/Cayley.Net) * **Rust** \(in early stage\): [https://github.com/shamansir/cayley-rust](https://github.com/shamansir/cayley-rust) * **Haskell**: [https://github.com/MichelBoucey/cayley-client](https://github.com/MichelBoucey/cayley-client) ================================================ FILE: docs/GizmoAPI.md.in ================================================ # Gizmo API ![Autogenerated file](https://img.shields.io/badge/file-generated-orange.svg) #AUTOGENERATED# ================================================ FILE: docs/README.md ================================================ # Cayley Documentation Welcome to the Cayley Manual! Cayley is an open-source graph database designed for ease of use and storing complex data. The manual introduces key concepts in Cayley, presents the query languages, and provides operational and administrative considerations and procedures as well as comprehensive reference section. ## Introduction * [Getting Started](getting-started.md) * [Installation](installation.md) * [Advanced Use](usage/advanced-use.md) * [UI Overview](usage/ui-overview.md) * [Project Locations](getting-involved/locations.md) ## Reference * [Glossary](getting-involved/glossary.md) * [Gizmo API](query-languages/gizmoapi.md) * [GraphQL](query-languages/graphql.md) * [MQL](query-languages/mql.md) * [HTTP](usage/http.md) * [GephiGraphStream](query-languages/gephigraphstream.md) ## Administrators * [Configuration](configuration.md) * [Migration](usage/migration.md) * [Usage as Container](deployment/container.md) * [Usage in Kubernetes](./k8s/k8s.md) ## Developers * [Libraries \(3rd party\)](usage/3rd-party-apis.md) * [Contributing](getting-involved/contributing.md) * [Quick Start As Go Library](usage/quickstart-as-lib.md) ================================================ FILE: docs/SUMMARY.md ================================================ # Table of contents * [Cayley Documentation](README.md) * [Getting Started](getting-started.md) * [Install Cayley](installation.md) * [Configuration](configuration.md) ## Usage * [Quickstart as Library](usage/quickstart-as-lib.md) * [Advanced Use](usage/advanced-use.md) * [HTTP Methods](usage/http.md) * [3rd-Party-APIs](usage/3rd-party-apis.md) * [UI Overview](usage/ui-overview.md) * [Migration](usage/migration.md) ## Query Languages * [Gizmo API](query-languages/gizmoapi.md) * [GraphQL Guide](query-languages/graphql.md) * [MQL Guide](query-languages/mql.md) * [Gephi GraphStream](query-languages/gephigraphstream.md) ## Getting Involved * [Glossary of Terms](getting-involved/glossary.md) * [Contributing](getting-involved/contributing.md) * [TODOs](getting-involved/todo.md) * [Locations of parts of Cayley](getting-involved/locations.md) ## Deployment * [Running in Docker](deployment/container.md) * [Running in Kubernetes](deployment/k8s-1.md) ## Tools * [Convert Linked Data files](tools/convert-linked-data-files.md) ================================================ FILE: docs/advanced-use.md ================================================ # Advanced Use ## Initialize A Graph Now that Cayley is downloaded \(or built\), let's create our database. `init` is the subcommand to set up a database and the right indices. You can set up a full [configuration file](configuration.md) if you'd prefer, but it will also work from the command line. Examples for each backend can be found in `store.address` format from [config file](configuration.md). Those two options \(db and dbpath\) are always going to be present. If you feel like not repeating yourself, setting up a configuration file for your backend might be something to do now. There's an example file, `cayley_example.yml` in the root directory. You can repeat the `--db (-i)` and `--dbpath (-a)` flags from here forward instead of the config flag, but let's assume you created `cayley_overview.yml` Note: when you specify parameters in the config file the config flags \(command line arguments\) are ignored. ## Load Data Into A Graph After the database is initialized we load the data. ```bash ./cayley load -c cayley_overview.yml -i data/testdata.nq ``` And wait. It will load. If you'd like to watch it load, you can run ```bash ./cayley load -c cayley_overview.yml -i data/testdata.nq --alsologtostderr=true ``` And watch the log output go by. If you plan to import a large dataset into Cayley and try multiple backends, it makes sense to first convert the dataset to Cayley-specific binary format by running: ```bash ./cayley conv -i dataset.nq.gz -o dataset.pq.gz ``` This will minimize parsing overhead on future imports and will compress dataset a bit better. ## Connect a REPL To Your Graph Now it's loaded. We can use Cayley now to connect to the graph. As you might have guessed, that command is: ```bash ./cayley repl -c cayley_overview.yml ``` Where you'll be given a `cayley>` prompt. It's expecting Gizmo/JS, but that can also be configured with a flag. New nodes and links can be added with the following command: ```bash cayley> :a subject predicate object label . ``` Removing links works similarly: ```bash cayley> :d subject predicate object . ``` This is great for testing, and ultimately also for scripting, but the real workhorse is the next step. Go ahead and give it a try: ```text // Simple math cayley> 2 + 2 // JavaScript syntax cayley> x = 2 * 8 cayley> x // See all the entities in this small follow graph. cayley> graph.Vertex().All() // See only dani. cayley> graph.Vertex("").All() // See who dani follows. cayley> graph.Vertex("").Out("").All() ``` ## Serve Your Graph Just as before: ```bash ./cayley http -c cayley_overview.yml ``` And you'll see a message not unlike ```bash listening on :64210, web interface at http://localhost:64210 ``` If you visit that address \(often, [http://localhost:64210](http://localhost:64210)\) you'll see the full web interface and also have a graph ready to serve queries via the [HTTP API](http.md) ### Access from other machines When you want to reach the API or UI from another machine in the network you need to specify the host argument: ```bash ./cayley http --config=cayley.cfg.overview --host=0.0.0.0:64210 ``` This makes it listen on all interfaces. You can also give it the specific the IP address you want Cayley to bind to. **Warning**: for security reasons you might not want to do this on a public accessible machine. ================================================ FILE: docs/api/swagger.yml ================================================ openapi: "3.0.0" info: description: "" version: "2.1.0" title: "Cayley API" license: name: "Apache 2.0" url: "http://www.apache.org/licenses/LICENSE-2.0.html" servers: - url: "http://{host}:{port}" variables: "host": default: "localhost" "port": default: "64210" tags: - name: "data" description: "Reading and writing data" - name: "queries" description: "Querying the graph" paths: /api/v2/formats: get: tags: - "data" summary: "Returns a list of supported data formats" description: "" operationId: "listFormats" responses: 200: description: "success" content: "application/json": schema: type: "object" properties: id: description: "unique name of the format" type: "string" read: description: "format is supported for loading quads" type: "boolean" write: description: "format is supported for exporting quads" type: "boolean" nodes: description: "format can be used to describe nodes" type: "boolean" ext: description: "typical file extensions for this format" type: "array" items: type: "string" mime: description: "typical content types for this format" type: "array" items: type: "string" binary: description: "format uses binary encoding" type: "boolean" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" /api/v2/read: get: tags: - "data" summary: "Reads all quads from the database" description: "" operationId: "readQuads" parameters: - name: "format" in: "query" description: "Data encoder to use for response. Overrides Accept header." required: false schema: type: "string" enum: - "nquads" - "jsonld" - "json" - "json-stream" - "pquads" - "graphviz" - "gml" - "graphml" default: "nquads" - name: "sub" in: "query" description: "Subjects to filter quads by" required: false schema: type: "string" - name: "pred" in: "query" description: "Predicates to filter quads by" required: false schema: type: "string" - name: "obj" in: "query" description: "Objects to filter quads by" required: false schema: type: "string" - name: "label" in: "query" description: "Labels to filter quads by" required: false schema: type: "string" - name: "iri" in: "query" description: "IRI format to use" required: false schema: type: "string" enum: ["short", "full"] responses: 200: description: "read successful" content: "application/n-quads": schema: $ref: "#/components/schemas/NQuads" "application/ld+json": schema: $ref: "#/components/schemas/JSONLD" "application/json": schema: $ref: "#/components/schemas/JsonQuads" "application/x-json-stream": schema: $ref: "#/components/schemas/JsonQuadsStream" "application/x-protobuf": schema: $ref: "#/components/schemas/PQuads" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" /api/v2/write: post: tags: - "data" summary: "Writes quads to the database" description: "" operationId: "writeQuads" requestBody: description: "File in one of formats specified in Content-Type." required: true content: "application/n-quads": schema: $ref: "#/components/schemas/NQuads" "application/ld+json": schema: $ref: "#/components/schemas/JSONLD" "application/json": schema: $ref: "#/components/schemas/JsonQuads" "application/x-json-stream": schema: $ref: "#/components/schemas/JsonQuadsStream" "application/x-protobuf": schema: $ref: "#/components/schemas/PQuads" parameters: - name: "format" in: "query" description: "Data decoder to use for request. Overrides Content-Type." required: false schema: type: "string" responses: 200: description: "write successful" content: application/json: schema: type: "object" properties: result: type: "string" description: "legacy success message" count: type: "integer" description: "number of quads received" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" /api/v2/node/delete: post: tags: - "data" summary: "Removes a node add all associated quads" description: "" operationId: "deleteNode" requestBody: description: "File in one of formats specified in Content-Type." required: true content: "application/n-quads": schema: $ref: "#/components/schemas/NQuadsNode" "application/json": schema: $ref: "#/components/schemas/JsonNode" "application/x-protobuf": schema: $ref: "#/components/schemas/PNode" parameters: - name: "format" in: "query" description: "Data decoder to use for request. Overrides Content-Type." required: false schema: type: "string" responses: 200: description: "delete successful" content: application/json: schema: type: "object" properties: result: type: "string" description: "legacy success message" count: type: "integer" description: "number of nodes deleted" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" /api/v2/delete: post: tags: - "data" summary: "Delete quads from the database" description: "" operationId: "deleteQuads" requestBody: description: "File in one of formats specified in Content-Type." required: true content: "application/n-quads": schema: $ref: "#/components/schemas/NQuads" "application/ld+json": schema: $ref: "#/components/schemas/JSONLD" "application/json": schema: $ref: "#/components/schemas/JsonQuads" "application/x-json-stream": schema: $ref: "#/components/schemas/JsonQuadsStream" "application/x-protobuf": schema: $ref: "#/components/schemas/PQuads" parameters: - name: "format" in: "query" description: "Data decoder to use for request. Overrides Content-Type." required: false schema: type: "string" responses: 200: description: "write successful" content: application/json: schema: type: "object" properties: result: type: "string" description: "legacy success message" count: type: "integer" description: "number of quads received" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" /api/v2/query: get: tags: - "queries" summary: "Query the graph" description: "" operationId: "query-get" parameters: - name: "lang" in: "query" description: "Query language to use" required: true schema: type: "string" enum: - "gizmo" - "graphql" - "mql" - "sexp" - name: "qu" in: "query" description: "Query text" required: true schema: type: "string" responses: 200: description: "query succesful" content: "application/json": schema: $ref: "#/components/schemas/QueryResult" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" post: tags: - "queries" summary: "Query the graph" description: "" operationId: "query" parameters: - name: "lang" in: "query" description: "Query language to use" required: true schema: type: "string" enum: - "gizmo" - "graphql" - "mql" - "sexp" requestBody: description: "Query text" required: true content: "*/*": schema: type: "string" examples: gizmo: summary: "Gizmo: first 10 nodes" value: "g.V().getLimit(10)" graphql: summary: "GraphQL: first 10 nodes" value: "{\n nodes(first: 10){\n id\n }\n}" responses: 200: description: "query succesful" content: "application/json": schema: $ref: "#/components/schemas/QueryResult" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" /api/v2/namespace-rules: get: tags: - "data" summary: "Reads all namespace rules from the database" description: "" operationId: "getNamespaceRules" responses: 200: description: "Success" content: "application/json": schema: type: "array" items: type: "object" properties: prefix: description: "Prefix of the namespace" type: "string" namespace: description: "The namespace prefixed" type: "string" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" post: tags: - "data" summary: "Registers new namespace rule to the database" description: "" operationId: "registerNamespaceRule" responses: 201: description: "Success" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" /gephi/gs: get: tags: - "queries" summary: "Gephi GraphStream endpoint" description: "" operationId: "gephiGraphStream" parameters: - name: "mode" in: "query" description: "Streamer mode" required: false schema: type: "string" enum: - "raw" - "nodes" default: "raw" - name: "limit" in: "query" description: "Limit the number of nodes or quads" required: false schema: type: "integer" - name: "sub" in: "query" description: "Subjects to filter quads by" required: false schema: type: "string" - name: "pred" in: "query" description: "Predicates to filter quads by" required: false schema: type: "string" - name: "obj" in: "query" description: "Objects to filter quads by" required: false schema: type: "string" - name: "label" in: "query" description: "Labels to filter quads by" required: false schema: type: "string" responses: 200: description: "success" content: "application/stream+json": schema: type: "string" format: "binary" description: "stream of JSON objects" default: description: "Unexpected error" content: application/json: schema: $ref: "#/components/schemas/Error" components: schemas: QueryResult: type: object properties: result: type: array nullable: true items: type: object NQuads: type: "string" format: "binary" example: | . . "cool_person" . . . . . "cool_person" . . . "cool_person" . . . "smart_person" . "smart_person" . NQuadsNode: type: "string" format: "binary" example: "" JSONLD: type: "string" format: "binary" example: { "@context": "http://schema.org/", "@type": "Person", "name": "Jane Doe", "jobTitle": "Professor", "telephone": "(425) 123-4567", "url": "http://www.janedoe.com", } externalDocs: url: "https://json-ld.org" JsonQuad: type: "object" properties: subject: type: "string" predicate: type: "string" object: type: "string" label: type: "string" JsonQuads: type: "array" items: type: "object" properties: subject: type: "string" predicate: type: "string" object: type: "string" label: type: "string" JsonNode: type: "string" JsonQuadsStream: type: "string" format: "binary" description: "stream of JsonQuad objects" PQuads: type: "string" format: "binary" description: "Cayley-specific binary encoding of quads based on protobuf" PNode: type: "string" format: "binary" description: "Cayley-specific binary encoding of node value based on protobuf" Error: type: "object" properties: error: type: "string" description: "error message" ================================================ FILE: docs/cayleyexport.md ================================================ # `cayleyexport` ``` cayleyexport ``` ## Synopsis The `cayleyexport` tool exports content from a Cayley deployment. See the [`cayleyimport`](cayleyimport.md) document for more information regarding [`cayleyimport`](cayleyimport.md), which provides the inverse “importing” capability. Run `cayleyexport` from the system command line, not the Cayley shell. ## Arguments ## Options ### `--help` Returns information on the options and use of **cayleyexport**. ### `--quiet` Runs **cayleyexport** in a quiet mode that attempts to limit the amount of output. ### `--uri=` Specify a resolvable URI connection string (enclose in quotes) to connect to the Cayley deployment. ``` --uri "http://host[:port]" ``` ### `--format=` Format to use for the exported data (if can not be detected defaults to JSON-LD) ### `--out=` Specifies the location and name of a file to export the data to. If you do not specify a file, **cayleyexport** writes data to the standard output (e.g. “stdout”). ================================================ FILE: docs/cayleyimport.md ================================================ # `cayleyimport` ``` cayleyimport ``` ## Synopsis The `cayleyimport` tool imports content created by [`cayleyexport`](cayleyexport.md), or potentially, another third-party export tool. See the [`cayleyexport`](cayleyexport.md) document for more information regarding [`cayleyexport`](cayleyexport.md), which provides the inverse “exporting” capability. Run `cayleyimport` from the system command line, not the Cayley shell. ## Arguments ### `file` Specifies the location and name of a file containing the data to import. If you do not specify a file, **cayleyimport** reads data from standard input (e.g. “stdin”). ## Options ### `--help` Returns information on the options and use of **cayleyimport**. ### `--quiet` Runs **cayleyimport** in a quiet mode that attempts to limit the amount of output. ### `--uri=` Specify a resolvable URI connection string (enclose in quotes) to connect to the Cayley deployment. ``` --uri "http://host[:port]" ``` ### `--format=` Format of the provided data (if can not be detected defaults to JSON-LD) ================================================ FILE: docs/configuration.md ================================================ # Configuration Cayley can be configured using configuration file written in YAML / JSON or by passing flags to the command line. By default. All command line flags take precedence over the configuration file. * [Recommended Configuration](configuration.md#Recommended-Configuration) * [Configuration Options](configuration.md#Configuration-Options) * [Store](configuration.md#Store) * [Per-Store Options](configuration.md#Per-Store-Options) * [Query](configuration.md#Query) * [Load](configuration.md#Load) * [Configuration File Location](configuration.md#Configuration-File-Location) ## Recommended Configuration By default, Cayley is using the `memstore` store. `memstore` works best for datasets that can fit into the memory of the machine and workloads which doesn't require persistency. For large datasets and/or workloads with require persistency it is recommended to use the `bolt` store. ## Configuration Options ### Store #### **`store.backend`** * Type: String * Default: `"memory"` Determines the type of the underlying database. Options include: * `memstore`: An in-memory store, based on an initial N-Quads file. Loses all changes when the process exits. **Key-Value backends** * `btree`: An in-memory store, used mostly to quickly verify KV backend functionality. * `leveldb`: A persistent on-disk store backed by [LevelDB](https://github.com/google/leveldb). * `bolt`: Stores the graph data on-disk in a [Bolt](https://github.com/boltdb/bolt) file. Uses more disk space and memory than LevelDB for smaller stores, but is often faster to write to and comparable for large ones, with faster average query times. **NoSQL backends** Slower, as it incurs network traffic, but multiple Cayley instances can disappear and reconnect at will, across a potentially horizontally-scaled store. * `mongo`: Stores the graph data and indices in a [MongoDB](https://www.mongodb.com/) instance. * `elastic`: Stores the graph data and indices in a [ElasticSearch](https://www.elastic.co/products/elasticsearch) instance. * `couch`: Stores the graph data and indices in a [CouchDB](http://couchdb.apache.org/) instance. * `pouch`: Stores the graph data and indices in a [PouchDB](https://pouchdb.com/). Requires building with [GopherJS](https://github.com/gopherjs/gopherjs). **SQL backends** * `postgres`: Stores the graph data and indices in a [PostgreSQL](https://www.postgresql.org) instance. * `cockroach`: Stores the graph data and indices in a [CockroachDB](https://www.cockroachlabs.com/product/cockroachdb/) cluster. * `mysql`: Stores the graph data and indices in a [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/) instance. * `sqlite`: Stores the graph data and indices in a [SQLite](https://www.sqlite.org) database. #### **`store.address`** * Type: String * Default: "" * Alias: `store.path` Where does the database actually live? Dependent on the type of database. For each datastore: * `memstore`: Path parameter is not supported. * `leveldb`: Directory to hold the LevelDB database files. * `bolt`: Path to the persistent single Bolt database file. * `mongo`: "hostname:port" of the desired MongoDB server. More options can be provided in [mgo](https://godoc.org/github.com/globalsign/mgo#Dial) address format. * `elastic`: `http://host:port` of the desired ElasticSearch server. * `couch`: `http://user:pass@host:port/dbname` of the desired CouchDB server. * `postgres`,`cockroach`: `postgres://[username:password@]host[:port]/database-name?sslmode=disable` of the PostgreSQL database and credentials. Sslmode is optional. More option available on [pq](https://godoc.org/github.com/lib/pq) page. * `mysql`: `[username:password@]tcp(host[:3306])/database-name` of the MqSQL database and credentials. More option available on [driver](https://github.com/go-sql-driver/mysql#dsn-data-source-name) page. * `sqlite`: `filepath` of the SQLite database. More options available on [driver](https://github.com/mattn/go-sqlite3#connection-string) page. #### **`store.read_only`** * Type: Boolean * Default: false If true, disables the ability to write to the database using the HTTP API \(will return a 400 for any write request\). Useful for testing or instances that shouldn't change. #### **`store.options`** * Type: Object See Per-Database Options, below. ### Per-Store Options The `store.options` object in the main configuration file contains any of these following options that change the behavior of the datastore. #### Memory No special options. #### LevelDB **`write_buffer_mb`** * Type: Integer * Default: 20 The size in MiB of the LevelDB write cache. Increasing this number allows for more/faster writes before syncing to disk. Default is 20, for large loads, a recommended value is 200+. **`cache_size_mb`** * Type: Integer * Default: 2 The size in MiB of the LevelDB block cache. Increasing this number uses more memory to maintain a bigger cache of quad blocks for better performance. #### Bolt **`nosync`** * Type: Boolean * Default: false Optionally disable syncing to disk per transaction. Nosync being true means much faster load times, but without consistency guarantees. #### Mongo **`database_name`** * Type: String * Default: "cayley" The name of the database within MongoDB to connect to. Manages its own collections and indices therein. #### PostgreSQL Postgres version 9.5 or greater is required. **`db_fill_factor`** * Type: Integer * Default: 50 Amount of empty space as a percentage to leave in the database when creating a table and inserting rows. See [PostgreSQL CreateTable](http://www.postgresql.org/docs/current/static/sql-createtable.html). **`local_optimize`** * Type: Boolean * Default: true Whether to skip checking quad store size. Connection pooling options used to configure the Go sql connection. Go defaults will be used when not specified. **`maxopenconnections`** * Type: Integer * Default: -1. **`maxidleconnections`** * Type: Integer * Default: -1. **`connmaxlifetime`** * Type: String * Default: "". #### Per-Replication Options The `replication_options` object in the main configuration file contains any of these following options that change the behavior of the replication manager. ### Query #### **`timeout`** * Type: Integer or String * Default: 30 The maximum length of time the Javascript runtime should run until cancelling the query and returning a 408 Timeout. When timeout is an integer is is interpreted as seconds, when it is a string it is [parsed](http://golang.org/pkg/time/#ParseDuration) as a Go time.Duration. A negative duration means no limit. ### Load #### **`load.ignore_missing`** * Type: Boolean * Default: false Optionally ignore missing quad on delete. #### **`load.ignore_duplicate`** * Type: Boolean * Default: false Optionally ignore duplicated quad on add. #### **`load.batch`** * Type: Integer * Default: 10000 The number of quads to buffer from a loaded file before writing a block of quads to the database. Larger numbers are good for larger loads. ## Configuration File Location Cayley looks in the following locations for the configuration file \(named `cayley.yml` or `cayley.json`\): * Command line flag * The environment variable `$CAYLEY_CFG` * Current directory * `$HOME/.cayley/` * `/etc/` ================================================ FILE: docs/container.md ================================================ # Container ## Running in Kubernetes To run Cayley in K8S check [this docs section](k8s/k8s.md). ## Running in a container A container exposing the HTTP API of Cayley is available. ### Running with default configuration Container is configured to use BoltDB as a backend by default. ```text docker run -p 64210:64210 -d ghcr.io/cayleygraph/cayley ``` New database will be available at [http://localhost:64210](http://localhost:64210). ### Custom configuration To run the container one must first setup a data directory that contains the configuration file and optionally contains persistent files \(i.e. a boltdb database file\). ```text mkdir data cp cayley_example.yml data/cayley.yml cp data/testdata.nq data/my_data.nq # initialize and serve database docker run -v $PWD/data:/data -p 64210:64210 -d ghcr.io/cayleygraph/cayley -c /data/cayley.yml --init -i /data/my_data.nq # serve existing database docker run -v $PWD/data:/data -p 64210:64210 -d ghcr.io/cayleygraph/cayley -c /data/cayley.yml ``` ### Other commands Container runs `cayley http` command by default. To run any other Cayley command reset the entry point for container: ```text docker run -v $PWD/data:/data ghcr.io/cayleygraph/cayley --entrypoint=cayley version ``` ================================================ FILE: docs/contributing.md ================================================ # Contributing ## Community Involvement Join our community on [discourse.cayley.io](https://discourse.cayley.io) or other [Locations](locations.md). ## Simply building Cayley Follow the instructions for running Cayley locally: ```text # clone project git clone https://github.com/cayleygraph/cayley cd cayley # Download dependencies go mod download # Update web files (optional) go run cmd/download_ui/download_ui.go ``` # build the binary go build ./cmd/cayley # try the generated binary ```bash ./cayley help ``` Give it a quick test with: ```text ./cayley repl -i data/testdata.nq ``` To run the web frontend, replace the "repl" command with "http" ```text ./cayley http -i data/testdata.nq ``` You can now open the WebUI in your browser: [http://127.0.0.1:64210](http://127.0.0.1:64210) ## Hacking on Cayley If you just want to build Cayley and check out the source, or use it as a library, a simple `go get github.com/cayleygraph/cayley` will work! But suppose you want to contribute back on your own fork \(and pull requests are welcome!\). A good way to do this is to set up your \$GOPATH and then... ```text git clone https://github.com/$GITHUBUSERNAME/cayley ``` ...where \$GITHUBUSERNAME is, well, your GitHub username :\) You'll probably want to add ```text cd cayley git remote add upstream http://github.com/cayleygraph/cayley ``` So that you can keep up with the latest changes by periodically running ```text git pull --rebase upstream ``` With that in place, that folder will reflect your local fork, be able to take changes from the official fork, and build in the Go style. For iterating, it can be helpful to, from the directory, run ```text go build ./cmd/cayley && ./cayley ``` Which will also resolve the relevant static content paths for serving HTTP. **Reminder:** add yourself to CONTRIBUTORS and AUTHORS. ## Running Unit Tests First, `cd` into the `cayley` project folder and run: ```text go test ./... ``` If you have a Docker installed, you can also run tests for remote backend implementations: ```text go test -tags docker ./... ``` If you have a Docker installed, you only want to run tests for a specific backend implementations eg. mongodb ```text go test -tags docker ./graph/nosql/mongo ``` Integration tests can be enabled with environment variable: ```text RUN_INTEGRATION=true go test ./... ``` ================================================ FILE: docs/convert-linked-data-files.md ================================================ --- description: >- Linked Data has multiple representations. The Cayley CLI includes a utility to convert Linked Data files from one format to another. --- # Convert Linked Data files ## Convert from one format to another ``` $ cayley convert -i data.jsonld -o data.nquads ``` `-i` is the input file to be converted. In this example it is a [JSON-LD](https://www.w3.org/TR/json-ld11/) file named `data.jsonld`. `-o` is the file to be created in the desired format. In this example it is a [N-Quads](https://www.w3.org/TR/n-quads/) file named `data.nquads`. ### Explicitly specify formats The formats of the input and output files are detected automatically by the file extension. In case a specific format should be used for input or output use `--load_format` and `--dump_format` respectively. ```text $ cayley convet -i data.jsonld -o data --dump_format pquads ``` `--dump_format` is set to the P-Quads format, a binary format used internally in Cayley. ================================================ FILE: docs/deployment/container.md ================================================ # Running in Docker ## Running in Kubernetes To run Cayley in K8S check [this docs section](k8s-1.md). ## Running in a container A container exposing the HTTP API of Cayley is available. ### Running with default configuration Container is configured to use BoltDB as a backend by default. ```text docker run -p 64210:64210 -d ghcr.io/cayleygraph/cayley ``` New database will be available at [http://localhost:64210](http://localhost:64210). ### Custom configuration To run the container one must first setup a data directory that contains the configuration file and optionally contains persistent files \(i.e. a boltdb database file\). ```text mkdir data cp cayley_example.yml data/cayley.yml cp data/testdata.nq data/my_data.nq # initialize and serve database docker run -v $PWD/data:/data -p 64210:64210 -d ghcr.io/cayleygraph/cayley -c /data/cayley.yml --init -i /data/my_data.nq # serve existing database docker run -v $PWD/data:/data -p 64210:64210 -d ghcr.io/cayleygraph/cayley -c /data/cayley.yml ``` ### Other commands Container runs `cayley http` command by default. To run any other Cayley command reset the entry point for container: ```text docker run -v $PWD/data:/data ghcr.io/cayleygraph/cayley --entrypoint=cayley version ``` ================================================ FILE: docs/deployment/k8s-1.md ================================================ # Running in Kubernetes Most examples requires Kubernetes 1.5+ and PersistentVolumes are configured. After running scripts namespace `cayley` will be created and service with the same name will be available in cluster. Service is of type `ClusterIP` by default. If you want to expose it, consider changing type to `LoadBalancer`. ## Single instance \(Bolt\) This is a simplest possible configuration: single Cayley instance with persistent storage, using Bolt as a backend. ```bash kubectl create -f ./docs/k8s/cayley-single.yml ``` ## Distributed \(MongoDB cluster\) This example is based on [thesandlord/mongo-k8s-sidecar](https://github.com/thesandlord/mongo-k8s-sidecar) and runs Cayley on top of 3-node Mongo cluster. ```bash kubectl create -f ./docs/k8s/cayley-mongo.yml ``` ## Distributed _TODO: PostgreSQL, CockroachDB_ ================================================ FILE: docs/docker-compose/docker-compose.mongo.yml ================================================ version: "2.2" services: cayley: image: cayleygraph/cayley command: http --db mongo --dbpath mongodb://mongo:27017 ports: - 64210:64210 mongo: image: mongo ports: - 27017:27017 ================================================ FILE: docs/faq.md ================================================ # Frequently Asked Questiones Coming Soon! Building the list at [https://discourse.cayley.io/t/faq-frequently-asked-questions/127](https://discourse.cayley.io/t/faq-frequently-asked-questions/127) ================================================ FILE: docs/gephigraphstream.md ================================================ # Gephi GraphStream Cayley supports graph visualisation in Gephi using GraphStream API. Enpoint can be accessed by adding URL `http://localhost:64210/gephi/gs` to Gephi GraphStream client. ## Options ### `limit` Default: `10000` Sets a maximal number of object that will be streamed. Depending on stream mode this could be either nodes or quads. Values less than 0 interpreted as "no limit". ### `mode` Sets streaming mode. Supported values: * `raw` \(default\) - all or selected quads * `nodes` - nodes with properties #### Raw mode In this mode Cayley directly streams selected quads to Gephi. Example URLs: `/gephi/gs?mode=raw&pred=&limit=-1` \(all quads\) `/gephi/gs?mode=raw&sub=&pred=,&limit=-1` \(links from `` via either `` or ``\) Parameters: * `limit` - maximal number of quads returned * `sub`,`pred`,`obj`,`label` - only show quads with specified values of Subject, Predicate, Object or Label This mode may be useful to visualize small subgraphs, or graphs without metadata such as types and properties. In case of later, large number of quads will be pointing to nodes describing common types or property names. For this kind of graphs `nodes` mode should be used. #### Nodes with properties Example URL: `/gephi/gs?mode=nodes&limit=-1` In this mode Cayley streams all nodes and links associated with them, but instead of streaming common quads such as types it will inline them as Gephi properties. Limit corresponds to a number of nodes returned. By default, all predicate will be streamed as in `raw` mode, except well-known predicates and ones with ` = "true"`. List of well-known predicates includes: * `` \(``\) * `` \(``\) * `` \(``\) To add custom predicates, write a special triple to database: ```text "true"^^ . ``` This allows to partition nodes by type or specific property values. Note: Only one value per predicate is supported for inlined properties. By default nodes will have random positions. To specify an exact position for specific node add `` and `` properties: ```text "10"^^ "-12.3"^^ ``` ================================================ FILE: docs/getting-involved/contributing.md ================================================ # Contributing ## Contributing ### Community Involvement Join our community on [discourse.cayley.io](https://discourse.cayley.io) or other [Locations](locations.md). ### Simply building Cayley If your version of Go < 1.13, you need to run: ```text export GO111MODULE=on ``` Follow the instructions for running Cayley locally: ```text # clone project git clone https://github.com/cayleygraph/cayley cd cayley # Download dependencies go mod download # Update web files (optional) go run cmd/download_ui/download_ui.go ``` ## Generate static files go modules packr2 ## build the binary go build ./cmd/cayley ## try the generated binary ```bash ./cayley help ``` Give it a quick test with: ```text ./cayley repl -i data/testdata.nq ``` To run the web frontend, replace the "repl" command with "http" ```text ./cayley http -i data/testdata.nq ``` You can now open the WebUI in your browser: [http://127.0.0.1:64210](http://127.0.0.1:64210) ### Hacking on Cayley First, you'll need Go [\(version 1.11.x or greater\)](https://golang.org/doc/install) and a Go workspace. This is outlined by the Go team at [http://golang.org/doc/code.html](http://golang.org/doc/code.html) and is sort of the official way of going about it. If your version of Go < 1.13, you need to run: ```text export GO111MODULE=on ``` If you just want to build Cayley and check out the source, or use it as a library, a simple `go get github.com/cayleygraph/cayley` will work! But suppose you want to contribute back on your own fork \(and pull requests are welcome!\). A good way to do this is to set up your $GOPATH and then... ```text mkdir -p $GOPATH/src/github.com/cayleygraph cd $GOPATH/src/github.com/cayleygraph git clone https://github.com/$GITHUBUSERNAME/cayley ``` ...where $GITHUBUSERNAME is, well, your GitHub username :\) You'll probably want to add ```text cd cayley git remote add upstream http://github.com/cayleygraph/cayley ``` So that you can keep up with the latest changes by periodically running ```text git pull --rebase upstream ``` With that in place, that folder will reflect your local fork, be able to take changes from the official fork, and build in the Go style. For iterating, it can be helpful to, from the directory, run ```text packr2 && go build ./cmd/cayley && ./cayley ``` Which will also resolve the relevant static content paths for serving HTTP. **Reminder:** add yourself to CONTRIBUTORS and AUTHORS. ### Running Unit Tests If your version of Go < 1.13, you need to run: ```text export GO111MODULE=on ``` First, `cd` into the `cayley` project folder and run: ```text go test ./... ``` If you have a Docker installed, you can also run tests for remote backend implementations: ```text go test -tags docker ./... ``` If you have a Docker installed, you only want to run tests for a specific backend implementations eg. mongodb ```text go test -tags docker ./graph/nosql/mongo ``` Integration tests can be enabled with environment variable: ```text RUN_INTEGRATION=true go test ./... ``` ================================================ FILE: docs/getting-involved/glossary.md ================================================ # Glossary of Terms _Note: this definitions in this glossary are sequenced so that they build on each other, one to the next, rather than alphabetically._ ## triple 1. a data entity composed of subject-predicate-object, like "Bob is 35" or "Bob knows Fred". \(A predicate in traditional grammar...is seen as a property that a subject has or is characterized by.\) [source](https://en.wikipedia.org/wiki/Triplestore) and \[source\]\([https://en.wikipedia.org/wiki/Predicate\_\(grammar\)\#Predicates\_in\_traditional\_grammar](https://en.wikipedia.org/wiki/Predicate_%28grammar%29#Predicates_in_traditional_grammar)\) ## triplestore 1. a purpose-built database for the storage and retrieval of triples... [source](https://en.wikipedia.org/wiki/Triplestore) ## quad 1. where triples have the form `{subject, predicate, object}`, quads would have a form along the lines of `{subject, predicate, object, context}` [source](https://en.wikipedia.org/wiki/Named_graph#Named_graphs_and_quads) 2. You can add context or extra values to triples that identifies them and makes it easy to define subgraphs, or named properties. [source](https://neo4j.com/blog/rdf-triple-store-vs-labeled-property-graph-difference/) 3. From [Cayley godoc](https://godoc.org/github.com/cayleygraph/quad#Quad): ```go type Quad struct { Subject Value `json:“subject”` Predicate Value `json:“predicate”` Object Value `json:“object”` Label Value `json:“label,omitempty”` } ``` ## link 1. Another name for a triple, since it "links" any two nodes. 2. Given the triple `{A, knows, C}` you would say in graph terminology that `A` and `C` are "vertices" while `knows` is an "edge". You would also say that `A`, `knows`, and `C` are all "nodes", and they are "linked" to one another by the triple. ## IRI 1. IRI is an RDF Internationalized Resource Identifier. [source](https://godoc.org/github.com/cayleygraph/quad#IRI) 2. An IRI \(Internationalized Resource Identifier\) within an RDF graph is a Unicode string that conforms to the syntax defined in RFC 3987. [source](https://www.w3.org/TR/rdf11-concepts/#h3_section-IRIs) 3. IRIs are a generalization of URIs that permits a wider range of Unicode characters. Every absolute URI and URL is an IRI, but not every IRI is an URI. [source](https://www.w3.org/TR/rdf11-concepts/#h3_section-IRIs) ## RDF 1. [Resource Description Framework](https://en.wikipedia.org/wiki/Resource_Description_Framework), basically a set of standards defined around quads 2. An RDF triple consists of three components: 1. the subject, which is an IRI or a blank node 2. the predicate, which is an IRI 3. the object, which is an IRI, a literal or a blank node [source](https://www.w3.org/TR/rdf11-concepts/#h3_section-triples) ## RDF store, quad store, named graph, semantic graph database 1. ...persisting RDF — storing it — became a thing, and these stores were called triple stores. Next they were called quad stores and included information about context and named graphs, then RDF stores, and most recently they call themselves “semantic graph database.” [source](https://neo4j.com/blog/rdf-triple-store-vs-labeled-property-graph-difference/) 2. Adding a name to the triple makes a "quad store" or named graph. [source](https://en.wikipedia.org/wiki/Triplestore#Related_database_types) ## Cayley 1. Cayley is a quad store that supports multiple storage backends. It supports multiple query languages for traversing and filtering the named graphs formed by its quads, and it has associated tooling such as a CLI, HTTP server, and so on. ## Gizmo 1. A [Gremlin/TinkerPop](http://tinkerpop.apache.org/)-inspired query language for Cayley. Looks a lot like JavaScript, the syntax is documented [here](https://github.com/cayleygraph/cayley/blob/master/docs/GizmoAPI.md#graphv). ## g.V\(\) 1. For Gremlin/TinkerPop, [g.V\(\) returns a list of all the vertices in the graph](http://tinkerpop.apache.org/docs/3.3.3/tutorials/gremlins-anatomy/#_graphtraversalsource) 2. `.v()` is for "Vertex" in Gizmo, and it is used like `pathObject = graph.Vertex([nodeId],[nodeId]...)` \(see \[\[path\|\#path\]\]\) ## inbound/outbound predicate 1. Inbound/outbound refers to the direction of a relation via a predicate. In the case of the triple "A follows B", "follows" is an outbound predicate for `A` and an inbound predicate for `B`. In/out predicates can be expressed in a query language, for example using the format `resultSet = subject.out(predicate)` to discover matching `Object`s. In the case of the triple "A follows B", `A.out(“follows”)` would return a set of nodes which contains `B`. An excellent example of this sort of query format is given in the Gremlin/TinkerPop homepage example: ```javascript What are the names of projects that were created by two friends? g.V().match( as(“a”).out(“knows”).as(“b”), as(“a”).out(“created”).as(“c”), as(“b”).out(“created”).as(“c”), as(“c”).in(“created”).count().is(2)). select(“c”).by(“name”) ``` ## direction 1. Direction specifies a node's position within a quad. [source](https://godoc.org/github.com/cayleygraph/quad#Direction) ```go const ( Any Direction = iota Subject Predicate Object Label ) ``` 2. Direction is passed to the `Get` method of a quad to access one of its four parts, see [quad.Get\(d Direction\) Value](https://godoc.org/github.com/cayleygraph/quad#Quad.Get) 3. The term "Direction" comes about from the concept of traversing a graph. Take for example the triple `{A, follows, B}` and supposing you "select" the predicate `follows`. Now you want to traverse the graph, so you move in the `Object` direction, and you now have `B` selected. Whereas the high-level [path](glossary.md#path) abstraction for queries uses inbound/outbound predicates to represent movement on the graph, the bottom-level [iterator](glossary.md#iterator) mechanic uses Direction. ## path 1. Paths are just a set of helpers to build a query, but they are not that good for building something more complex. You can try using [Shapes](glossary.md#shape) for this - it will give you a full control of what the query actually does. [source](https://discourse.cayley.io/t/a-variety-of-questions/1183/2) 2. Path represents either a morphism \(a pre-defined path stored for later use\), or a concrete path, consisting of a morphism and an underlying QuadStore. [source](https://godoc.org/github.com/cayleygraph/cayley/query/path#Path) 3. Underlying code: ```go type Path struct { stack []morphism qs graph.QuadStore baseContext pathContext } type morphism struct { IsTag bool Reversal func(*pathContext) (morphism, *pathContext) Apply applyMorphism tags []string } type applyMorphism func(shape.Shape, *pathContext) (shape.Shape, *pathContext) ``` So, as previously stated, the [path](https://godoc.org/github.com/cayleygraph/cayley/query/path) package is just helper methods on top of the [shape](https://godoc.org/github.com/cayleygraph/cayley/graph/shape) package. ## morphism 1. Morphism is basically a path that is not attached to any particular quadstore or a particular starting point in the graph. Morphisms are meant to be used as a query part that can be applied to other queries to follow a path specified in the Morphism. A good example will be a `FollowRecursive` function that will apply a single morphism multiple times to get to all nodes that can be traversed recursively. [source](https://discourse.cayley.io/t/a-variety-of-questions/1183/2) ## iterator 1. So a graph query is roughly represented as a tree of iterators – things that implement graph.Iterator. An iterator is \(loosely\) a stand-in for a set of things that match a particular portion of the graph. [source](https://discourse.cayley.io/t/7-7-14-question-about-iterator/62) ## subiterator 1. So a graph query is roughly represented as a tree of iterators...Evaluation is merely calling Next\(\) repeatedly on the iterator at the top of the tree. Subiterators, then, are the branches and leaves of the tree. [source](https://discourse.cayley.io/t/7-7-14-question-about-iterator/62) 2. Example of converting the Cayley-Gremlin-Go-API query `g.V(“B”).In(“follows”).All()` into an iterator tree: * **HasA** \(subject\) – gets the things in the subject field for: * **And** – the intersection of: * **LinksTo \(predicate\)** links that have the predicate of…: * Fixed iterator containing “follows” – … just the node “follows”. * **LinksTo \(object\)** links that have the object field of: * Fixed iterator containing “B” – … just the node “B” ## LinkTo iterator 1. A LinksTo takes a subiterator of nodes, and contains an iteration of links which "link to" those nodes in a given direction. ... Can be seen as the dual of the HasA iterator. [source](https://github.com/cayleygraph/cayley/blob/1f53d04893ea9b2736e9b2277bbba3f47b88711a/graph/iterator/linksto.go#L17) * Next\(\)ing a LinksTo is straightforward -- iterate through all links to things in the subiterator, and then advance the subiterator, and do it again. * To restate in pseudo-code; `results` is what would be returned in successive `Next()` calls: ```go var results []quad.Quad for _, node := range linkTo.subIterator { for _, quad := range allQuads { if quad.Get(linkTo.direction) == node { results = append(results, quad) } } } ``` * Contains\(\)ing a LinksTo means, given a link, take the direction we care about and check if it's in our subiterator. * To restate in pseudo-code: ```go for _, node := range linkTo.subIterator { if theLink.Get(linkTo.direction) == node { return true } } return false ``` ## HasA iterator 1. The HasA takes a subiterator of links, and acts as an iterator of nodes in the given direction. The name comes from the idea that a "link HasA subject" or a "link HasA predicate". [source](https://github.com/cayleygraph/cayley/blob/41bf496d9dfe622b385c1482789480df8b106472/graph/iterator/hasa.go#L17) * Next\(\), [We have a subiterator we can get a value from, and we can take that resultant quad, pull our direction out of it, and return that.](https://github.com/cayleygraph/cayley/blob/41bf496d9dfe622b385c1482789480df8b106472/graph/iterator/hasa.go#L206) ```go var results []quad.Value for _, quad := range hasA.subIterator { results = append(results, quad.Get(hasA.direction)) } ``` * Contains\(\) ```go for _, quad := range hasA.subIterator { if quad.Get(hasA.direction) == theNode { return true } } return false ``` ## shape 1. Shape represent a query tree shape. [source](https://godoc.org/github.com/cayleygraph/cayley/graph/shape#Shape) ```go type Shape interface { BuildIterator(qs graph.QuadStore) graph.Iterator Optimize(r Optimizer) (Shape, bool) } ``` 2. This is the most interesting part of the query system - it describes how exactly the query looks like. ... This package also describes different query optimizations that are not specific to a backend. ... You can write a query using either Paths, Shapes or raw Iterators... [source](https://discourse.cayley.io/t/a-variety-of-questions/1183/2) 3. A Shape seems to be an abstract representation of a query, a level above Iterators and a level below Paths. You can perform various operations on it \(traverse inbound/outbound predicates, find unions and intersections, etc.\) and most importantly build a tree of Iterators from it, which will do the mechanical act of processing quads to find results. ## token 1. In the context of a [quad store](https://godoc.org/github.com/cayleygraph/cayley/graph#QuadStore), a [graph.Value](https://godoc.org/github.com/cayleygraph/cayley/graph#Value). However the backend wishes to implement it, a Value is merely a token to a quad or a node that the backing store itself understands, and the base iterators pass around. For example, in a very traditional, graphd-style graph, these are int64s \(guids of the primitives\). In a very direct sort of graph, these could be pointers to structs, or merely quads, or whatever works best for the backing store. ## reification 1. “With reification, we create a metagraph on top of our graph that represents the statement that we have here. We create a new node that represents a statement and points at the subject...” [source](https://neo4j.com/blog/rdf-triple-store-vs-labeled-property-graph-difference/) 2. Reifying a relationship means viewing it as an entity. The purpose of reifying a relationship is to make it explicit, when additional information needs to be added to it. Viewing a relationship as an entity, one can say that the entity reifies the relationship. This is called reification of a relationship. Like any other entity, it must be an instance of an entity type. [source](https://en.wikipedia.org/wiki/Reification_) ================================================ FILE: docs/getting-involved/locations.md ================================================ # Locations of parts of Cayley ## Community * Where is the beating heart of this community? [https://discourse.cayley.io/](https://discourse.cayley.io/) * Where is the mailing list? [https://discourse.cayley.io/](https://discourse.cayley.io/) \(enable mailing list mode under options\) * Where is the chat room? [cayleygraph.slack.com](https://cayleygraph.slack.com) -- Invite [here](https://cayley-slackin.herokuapp.com/) * Where should I start contributing? [Contributing.md](contributing.md) \(Also: [How to get involved!](https://discourse.cayley.io/t/how-to-get-involved/44)\) * Where is the site-content? [https://github.com/cayleygraph/site-content](https://github.com/cayleygraph/site-content) * Where are issues? [https://github.com/cayleygraph/cayley/issues](https://github.com/cayleygraph/cayley/issues) * Where is the graph source code? [https://github.com/cayleygraph/cayley](https://github.com/cayleygraph/cayley) * Where are pull requests? [https://github.com/cayleygraph/cayley/pulls](https://github.com/cayleygraph/cayley/pulls) * Where is the wiki? It was at [https://github.com/cayleygraph/cayley/wiki](https://github.com/cayleygraph/cayley/wiki), but we are deprecating it. * Where is the \(old\) mailing list? It was at [https://groups.google.com/forum/\#!forum/cayley-users](https://groups.google.com/forum/#!forum/cayley-users), but we are deprecating it for this forum \(and its mailing list mode\). ================================================ FILE: docs/getting-involved/todo.md ================================================ # TODOs The main source of our TODO list is our [Github Issues](https://github.com/cayleygraph/cayley/issues), so we are going to try to avoid duplicating them here. ## Anything marked "TODO" in the code. Usually something that should be taken care of. ================================================ FILE: docs/getting-started.md ================================================ # Getting Started This guide will take you through starting a graph based on provided data. ## Prerequisites This tutorial requires you to be connected to **local Cayley installation**. For more information on installing Cayley locally, see [Install Cayley](installation.md). ## Start Cayley ```bash cayley http ``` You should see: ```text Cayley version: 0.7.7 (dev snapshot) using backend "memstore" listening on 127.0.0.1:64210, web interface at http://127.0.0.1:64210 ``` You can now open the web-interface on: [localhost:64210](http://localhost:64210/). Cayley is configured by default to run in memory \(That's what `backend memstore` means\). To change the configuration see the documentation for [Configuration File](configuration.md) or run `cayley http --help`. For more information about the UI see: [UI Overview](usage/ui-overview.md) ## Run with sample data ### Download sample data [Sample Data](https://github.com/cayleygraph/cayley/raw/master/data/30kmoviedata.nq.gz) ### Run Cayley ```bash cayley http --load 30kmoviedata.nq.gz ``` ## Query Data Using the 30kmoviedata.nq dataset from above, let's walk through some simple queries: ### Query all vertices in the graph To select all vertices in the graph call, limit to 5 first results. `g` and `V` are synonyms for `graph` and `Vertex` respectively, as they are quite common. ```javascript g.V().getLimit(5); ``` ### Match a property of a vertex Find vertex with property "Humphrey Bogart" ```javascript g.V() .has("", "Humphrey Bogart") .all(); ``` You may start to notice a pattern here: with Gizmo, the query lines tend to: Start somewhere in the graph \| Follow a path \| Run the query with "all" or "getLimit" ### Match a complex path Get the list of actors in the film ```javascript g.V() .has("", "Casablanca") .out("") .out("") .out("") .all(); ``` ### Match This is starting to get long. Let's use a Morphism, a pre-defined path stored in a variable, as our linkage ```javascript var filmToActor = g .Morphism() .out("") .out(""); g.V() .has("", "Casablanca") .follow(filmToActor) .out("") .all(); ``` To learn more about querying see [Gizmo Documentation](query-languages/gizmoapi.md) ================================================ FILE: docs/gizmoapi.md ================================================ # Gizmo API ![Autogenerated file](https://img.shields.io/badge/file-generated-orange.svg) ## The `graph` object Name: `graph`, Alias: `g` This is the only special object in the environment, generates the query objects. Under the hood, they're simple objects that get compiled to a Go iterator tree when executed. ### `graph.addDefaultNamespaces()` AddDefaultNamespaces register all default namespaces for automatic IRI resolution. ### `graph.addNamespace(pref, ns)` AddNamespace associates prefix with a given IRI namespace. ### `graph.emit(*)` Emit adds data programmatically to the JSON result list. Can be any JSON type. ```javascript g.emit({ name: "bob" }); // push {"name":"bob"} as a result ``` ### `graph.loadNamespaces()` LoadNamespaces loads all namespaces saved to graph. ### `graph.M()` M is a shorthand for Morphism. ### `graph.Morphism()` Morphism creates a morphism path object. Unqueryable on it's own, defines one end of the path. Saving these to variables with ```javascript var shorterPath = graph .Morphism() .out("foo") .out("bar"); ``` is the common use case. See also: path.follow\(\), path.followR\(\). ### `graph.IRI(s)` Uri creates an IRI values from a given string. ### `graph.V(*)` V is a shorthand for Vertex. ### `graph.Vertex([nodeId],[nodeId]...)` Vertex starts a query path at the given vertex/vertices. No ids means "all vertices". Arguments: * `nodeId` \(Optional\): A string or list of strings representing the starting vertices. Returns: Path object ## Path object Both `.Morphism()` and `.Vertex()` create path objects, which provide the following traversal methods. Note that `.Vertex()` returns a query object, which is a subclass of path object. For these examples, suppose we have the following graph: ```text +-------+ +------+ | alice |----- ->| fred |<-- +-------+ \---->+-------+-/ +------+ \-+-------+ ----->| #bob# | | |*emily*| +---------+--/ --->+-------+ | +-------+ | charlie | / v +---------+ / +--------+ \--- +--------+ |*#greg#*| \-->| #dani# |------------>+--------+ +--------+ ``` Where every link is a `` relationship, and the nodes with an extra `#` in the name have an extra `` link. As in, ```text -- --> "cool_person" ``` Perhaps these are the influencers in our community. So too are extra `*`s in the name -- these are our smart people, according to the `` label, eg, the quad: ```text "smart_person" . ``` ### `path.all()` All executes the query and adds the results, with all tags, as a string-to-string \(tag to node\) map in the output set, one for each path that a traversal could take. ### `path.and(path)` And is an alias for Intersect. ### `path.as(tags)` As is an alias for Tag. ### `path.back([tag])` Back returns current path to a set of nodes on a given tag, preserving all constraints. If still valid, a path will now consider their vertex to be the same one as the previously tagged one, with the added constraint that it was valid all the way here. Useful for traversing back in queries and taking another route for things that have matched so far. Arguments: * `tag`: A previous tag in the query to jump back to. Example: ```javascript // Start from all nodes, save them into start, follow any status links, // jump back to the starting node, and find who follows them. Return the result. // Results are: // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""} g.V() .Tag("start") .out("") .back("start") .in("") .all(); ``` ### `path.both([predicatePath], [tags])` Both follow the predicate in either direction. Same as Out or In. Example: ```javascript // Find all followers/followees of fred. Returns bob, emily and greg g.V("") .both("") .all(); ``` ### `path.count()` Count returns a number of results and returns it as a value. Example: ```javascript // Save count as a variable var n = g.V().count(); // Send it as a query result g.emit(n); ``` ### `path.difference(path)` Difference is an alias for Except. ### `path.except(path)` Except removes all paths which match query from current path. In a set-theoretic sense, this is \(A - B\). While `g.V().except(path)` to achieve `U - B = !B` is supported, it's often very slow. Example: ```javascript var cFollows = g.V("").out(""); var dFollows = g.V("").out(""); // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob. cFollows.except(dFollows).all(); // The set (dani) -- what charlie follows that dani does not also follow. // Equivalently, g.V("").out("").except(g.V("").out("")).all() ``` ### `path.filter(args)` Filter applies constraints to a set of nodes. Can be used to filter values by range or match strings. #### `path.filter(regex(expression, includeIRIs))` Filters by match a regular expression \([syntax](https://github.com/google/re2/wiki/Syntax)\). By default works only on literals unless includeEntities is set to `true`. ### `path.follow(path)` Follow is the way to use a path prepared with Morphism. Applies the path chain on the morphism object to the current path. Starts as if at the g.M\(\) and follows through the morphism path. Example: ```javascript var friendOfFriend = g .Morphism() .out("") .out(""); // Returns the followed people of who charlie follows -- a simplistic "friend of my friend" // and whether or not they have a "cool" status. Potential for recommending followers abounds. // Returns bob and greg g.V("") .follow(friendOfFriend) .has("", "cool_person") .all(); ``` ### `path.followR(path)` FollowR is the same as Follow but follows the chain in the reverse direction. Flips "In" and "Out" where appropriate, the net result being a virtual predicate followed in the reverse direction. Starts at the end of the morphism and follows it backwards \(with appropriate flipped directions\) to the g.M\(\) location. Example: ```javascript var friendOfFriend = g .Morphism() .out("") .out(""); // Returns the third-tier of influencers -- people who follow people who follow the cool people. // Returns charlie (from bob), charlie (from greg), bob and emily g.V() .has("", "cool_person") .followR(friendOfFriend) .all(); ``` ### `path.followRecursive(*)` FollowRecursive is the same as Follow but follows the chain recursively. Starts as if at the g.M\(\) and follows through the morphism path multiple times, returning all nodes encountered. Example: ```javascript var friend = g.Morphism().out(""); // Returns all people in Charlie's network. // Returns bob and dani (from charlie), fred (from bob) and greg (from dani). g.V("") .followRecursive(friend) .all(); ``` ### `path.forEach(callback) or (limit, callback)` ForEach calls callback\(data\) for each result, where data is the tag-to-string map as in All case. Arguments: * `limit` \(Optional\): An integer value on the first `limit` paths to process. * `callback`: A javascript function of the form `function(data)` Example: ```javascript // Simulate query.all().all() graph.V("").ForEach(function(d) { g.emit(d); }); ``` ### `path.getLimit(limit)` GetLimit is the same as All, but limited to the first N unique nodes at the end of the path, and each of their possible traversals. ### `path.has(predicate, object)` Has filters all paths which are, at this point, on the subject for the given predicate and object, but do not follow the path, merely filter the possible paths. Usually useful for starting with all nodes, or limiting to a subset depending on some predicate/value pair. Arguments: * `predicate`: A string for a predicate node. * `object`: A string for a object node or a set of filters to find it. Example: ```javascript // Start from all nodes that follow bob -- results in alice, charlie and dani g.V() .has("", "") .all(); // People charlie follows who then follow fred. Results in bob. g.V("") .out("") .has("", "") .all(); // People with friends who have names sorting lower then "f". g.V() .has("", gt("")) .all(); ``` ### `path.hasR(*)` HasR is the same as Has, but sets constraint in reverse direction. ### `path.in([predicatePath], [tags])` In is inverse of Out. Starting with the nodes in `path` on the object, follow the quads with predicates defined by `predicatePath` to their subjects. Arguments: * `predicatePath` \(Optional\): One of: * null or undefined: All predicates pointing into this node * a string: The predicate name to follow into this node * a list of strings: The predicates to follow into this node * a query path object: The target of which is a set of predicates to follow. * `tags` \(Optional\): One of: * null or undefined: No tags * a string: A single tag to add the predicate used to the output set. * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. Example: ```javascript // Find the cool people, bob, dani and greg g.V("cool_person") .in("") .all(); // Find who follows bob, in this case, alice, charlie, and dani g.V("") .in("") .all(); // Find who follows the people emily follows, namely, bob and emily g.V("") .out("") .in("") .all(); ``` ### `path.inPredicates()` InPredicates gets the list of predicates that are pointing in to a node. Example: ```javascript // bob only has "" predicates pointing inward // returns "" g.V("") .inPredicates() .all(); ``` ### `path.intersect(path)` Intersect filters all paths by the result of another query path. This is essentially a join where, at the stage of each path, a node is shared. Example: ```javascript var cFollows = g.V("").out(""); var dFollows = g.V("").out(""); // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob. cFollows.intersect(dFollows).all(); // Equivalently, g.V("").out("").And(g.V("").out("")).all() ``` ### `path.is(node, [node..])` Filter all paths to ones which, at this point, are on the given node. Arguments: * `node`: A string for a node. Can be repeated or a list of strings. Example: ```javascript // Starting from all nodes in the graph, find the paths that follow bob. // Results in three paths for bob (from alice, charlie and dani).all() g.V() .out("") .is("") .all(); ``` ### `path.labelContext([labelPath], [tags])` LabelContext sets \(or removes\) the subgraph context to consider in the following traversals. Affects all In\(\), Out\(\), and Both\(\) calls that follow it. The default LabelContext is null \(all subgraphs\). Arguments: * `predicatePath` \(Optional\): One of: * null or undefined: In future traversals, consider all edges, regardless of subgraph. * a string: The name of the subgraph to restrict traversals to. * a list of strings: A set of subgraphs to restrict traversals to. * a query path object: The target of which is a set of subgraphs. * `tags` \(Optional\): One of: * null or undefined: No tags * a string: A single tag to add the last traversed label to the output set. * a list of strings: Multiple tags to use as keys to save the label used to the output set. Example: ```javascript // Find the status of people Dani follows g.V("") .out("") .out("") .all(); // Find only the statuses provided by the smart_graph g.V("") .out("") .labelContext("") .out("") .all(); // Find all people followed by people with statuses in the smart_graph. g.V() .labelContext("") .in("") .labelContext(null) .in("") .all(); ``` ### `path.labels()` Labels gets the list of inbound and outbound quad labels ### `path.limit(limit)` Limit limits a number of nodes for current path. Arguments: * `limit`: A number of nodes to limit results to. Example: ```javascript // Start from all nodes that follow bob, and limit them to 2 nodes -- results in alice and charlie g.V() .has("", "") .limit(2) .all(); ``` ### `path.map(*)` Map is a alias for ForEach. ### `path.or(path)` Or is an alias for Union. ### `path.out([predicatePath], [tags])` Out is the work-a-day way to get between nodes, in the forward direction. Starting with the nodes in `path` on the subject, follow the quads with predicates defined by `predicatePath` to their objects. Arguments: * `predicatePath` \(Optional\): One of: * null or undefined: All predicates pointing out from this node * a string: The predicate name to follow out from this node * a list of strings: The predicates to follow out from this node * a query path object: The target of which is a set of predicates to follow. * `tags` \(Optional\): One of: * null or undefined: No tags * a string: A single tag to add the predicate used to the output set. * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. Example: ```javascript // The working set of this is bob and dani g.V("") .out("") .all(); // The working set of this is fred, as alice follows bob and bob follows fred. g.V("") .out("") .out("") .all(); // Finds all things dani points at. Result is bob, greg and cool_person g.V("") .out() .all(); // Finds all things dani points at on the status linkage. // Result is bob, greg and cool_person g.V("") .out(["", ""]) .all(); // Finds all things dani points at on the status linkage, given from a separate query path. // Result is {"id": "cool_person", "pred": ""} g.V("") .out(g.V(""), "pred") .all(); ``` ### `path.outPredicates()` OutPredicates gets the list of predicates that are pointing out from a node. Example: ```javascript // bob has "" and "" edges pointing outwards // returns "", "" g.V("") .outPredicates() .all(); ``` ### `path.save(predicate, tag)` Save saves the object of all quads with predicate into tag, without traversal. Arguments: * `predicate`: A string for a predicate node. * `tag`: A string for a tag key to store the object node. Example: ```javascript // Start from dani and bob and save who they follow into "target" // Returns: // {"id" : "", "target": "" }, // {"id" : "", "target": "" }, // {"id" : "", "target": "" } g.V("", "") .save("", "target") .all(); ``` ### `path.saveInPredicates(tag)` SaveInPredicates tags the list of predicates that are pointing in to a node. Example: ```javascript // bob only has "" predicates pointing inward // returns {"id":"", "pred":""} g.V("") .saveInPredicates("pred") .all(); ``` ### `path.saveOpt(*)` SaveOpt is the same as Save, but returns empty tags if predicate does not exists. ### `path.saveOptR(*)` SaveOptR is the same as SaveOpt, but tags values via reverse predicate. ### `path.saveOutPredicates(tag)` SaveOutPredicates tags the list of predicates that are pointing out from a node. Example: ```javascript // bob has "" and "" edges pointing outwards // returns {"id":"", "pred":""} g.V("") .saveInPredicates("pred") .all(); ``` ### `path.saveR(*)` SaveR is the same as Save, but tags values via reverse predicate. ### `path.skip(offset)` Skip skips a number of nodes for current path. Arguments: * `offset`: A number of nodes to skip. Example: ```javascript // Start from all nodes that follow bob, and skip 2 nodes -- results in dani g.V() .has("", "") .skip(2) .all(); ``` ### `path.tag(tags)` Tag saves a list of nodes to a given tag. In order to save your work or learn more about how a path got to the end, we have tags. The simplest thing to do is to add a tag anywhere you'd like to put each node in the result set. Arguments: * `tag`: A string or list of strings to act as a result key. The value for tag was the vertex the path was on at the time it reached "Tag" Example: ```javascript // Start from all nodes, save them into start, follow any status links, and return the result. // Results are: // {"id": "cool_person", "start": ""}, // {"id": "cool_person", "start": ""}, // {"id": "cool_person", "start": ""}, // {"id": "smart_person", "start": ""}, // {"id": "smart_person", "start": ""} g.V() .tag("start") .out("") .all(); ``` ### `path.tagArray(*)` TagArray is the same as ToArray, but instead of a list of top-level nodes, returns an Array of tag-to-string dictionaries, much as All would, except inside the JS environment. Example: ```javascript // bobTags contains an Array of followers of bob (alice, charlie, dani). var bobTags = g .V("") .tag("name") .in("") .tagArray(); // nameValue should be the string "" var nameValue = bobTags[0]["name"]; ``` ### `path.tagValue()` TagValue is the same as TagArray, but limited to one result node. Returns a tag-to-string map. ### `path.toArray(*)` ToArray executes a query and returns the results at the end of the query path as an JS array. Example: ```javascript // bobFollowers contains an Array of followers of bob (alice, charlie, dani). var bobFollowers = g .V("") .in("") .toArray(); ``` ### `path.toValue()` ToValue is the same as ToArray, but limited to one result node. ### `path.union(path)` Union returns the combined paths of the two queries. Notice that it's per-path, not per-node. Once again, if multiple paths reach the same destination, they might have had different ways of getting there \(and different tags\). See also: `path.Tag()` Example: ```javascript var cFollows = g.V("").out(""); var dFollows = g.V("").out(""); // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob (from charlie), dani, bob (from dani), and greg. cFollows.union(dFollows).all(); ``` ### `path.unique()` Unique removes duplicate values from the path. ### `path.order()` Order returns values from the path in ascending order. ================================================ FILE: docs/glossary.md ================================================ # Glossary of Terms _Note: this definitions in this glossary are sequenced so that they build on each other, one to the next, rather than alphabetically._ ## triple 1. a data entity composed of subject-predicate-object, like "Bob is 35" or "Bob knows Fred". \(A predicate in traditional grammar...is seen as a property that a subject has or is characterized by.\) [source](https://en.wikipedia.org/wiki/Triplestore) and \[source\]\([https://en.wikipedia.org/wiki/Predicate\_\(grammar\)\#Predicates\_in\_traditional\_grammar](https://en.wikipedia.org/wiki/Predicate_%28grammar%29#Predicates_in_traditional_grammar)\) ## triplestore 1. a purpose-built database for the storage and retrieval of triples... [source](https://en.wikipedia.org/wiki/Triplestore) ## quad 1. where triples have the form `{subject, predicate, object}`, quads would have a form along the lines of `{subject, predicate, object, context}` [source](https://en.wikipedia.org/wiki/Named_graph#Named_graphs_and_quads) 2. You can add context or extra values to triples that identifies them and makes it easy to define subgraphs, or named properties. [source](https://neo4j.com/blog/rdf-triple-store-vs-labeled-property-graph-difference/) 3. From [Cayley godoc](https://godoc.org/github.com/cayleygraph/quad#Quad): ```go type Quad struct { Subject Value `json:“subject”` Predicate Value `json:“predicate”` Object Value `json:“object”` Label Value `json:“label,omitempty”` } ``` ## link 1. Another name for a triple, since it "links" any two nodes. 2. Given the triple `{A, knows, C}` you would say in graph terminology that `A` and `C` are "vertices" while `knows` is an "edge". You would also say that `A`, `knows`, and `C` are all "nodes", and they are "linked" to one another by the triple. ## IRI 1. IRI is an RDF Internationalized Resource Identifier. [source](https://godoc.org/github.com/cayleygraph/quad#IRI) 2. An IRI \(Internationalized Resource Identifier\) within an RDF graph is a Unicode string that conforms to the syntax defined in RFC 3987. [source](https://www.w3.org/TR/rdf11-concepts/#h3_section-IRIs) 3. IRIs are a generalization of URIs that permits a wider range of Unicode characters. Every absolute URI and URL is an IRI, but not every IRI is an URI. [source](https://www.w3.org/TR/rdf11-concepts/#h3_section-IRIs) ## RDF 1. [Resource Description Framework](https://en.wikipedia.org/wiki/Resource_Description_Framework), basically a set of standards defined around quads 2. An RDF triple consists of three components: 1. the subject, which is an IRI or a blank node 2. the predicate, which is an IRI 3. the object, which is an IRI, a literal or a blank node [source](https://www.w3.org/TR/rdf11-concepts/#h3_section-triples) ## RDF store, quad store, named graph, semantic graph database 1. ...persisting RDF — storing it — became a thing, and these stores were called triple stores. Next they were called quad stores and included information about context and named graphs, then RDF stores, and most recently they call themselves “semantic graph database.” [source](https://neo4j.com/blog/rdf-triple-store-vs-labeled-property-graph-difference/) 2. Adding a name to the triple makes a "quad store" or named graph. [source](https://en.wikipedia.org/wiki/Triplestore#Related_database_types) ## Cayley 1. Cayley is a quad store that supports multiple storage backends. It supports multiple query languages for traversing and filtering the named graphs formed by its quads, and it has associated tooling such as a CLI, HTTP server, and so on. ## Gizmo 1. A [Gremlin/TinkerPop](http://tinkerpop.apache.org/)-inspired query language for Cayley. Looks a lot like JavaScript, the syntax is documented [here](https://github.com/cayleygraph/cayley/blob/master/docs/GizmoAPI.md#graphv). ## g.V\(\) 1. For Gremlin/TinkerPop, [g.V\(\) returns a list of all the vertices in the graph](http://tinkerpop.apache.org/docs/3.3.3/tutorials/gremlins-anatomy/#_graphtraversalsource) 2. `.v()` is for "Vertex" in Gizmo, and it is used like `pathObject = graph.Vertex([nodeId],[nodeId]...)` \(see \[\[path\|\#path\]\]\) ## inbound/outbound predicate 1. Inbound/outbound refers to the direction of a relation via a predicate. In the case of the triple "A follows B", "follows" is an outbound predicate for `A` and an inbound predicate for `B`. In/out predicates can be expressed in a query language, for example using the format `resultSet = subject.out(predicate)` to discover matching `Object`s. In the case of the triple "A follows B", `A.out(“follows”)` would return a set of nodes which contains `B`. An excellent example of this sort of query format is given in the Gremlin/TinkerPop homepage example: ```javascript What are the names of projects that were created by two friends? g.V().match( as(“a”).out(“knows”).as(“b”), as(“a”).out(“created”).as(“c”), as(“b”).out(“created”).as(“c”), as(“c”).in(“created”).count().is(2)). select(“c”).by(“name”) ``` ## direction 1. Direction specifies a node's position within a quad. [source](https://godoc.org/github.com/cayleygraph/quad#Direction) ```go const ( Any Direction = iota Subject Predicate Object Label ) ``` 2. Direction is passed to the `Get` method of a quad to access one of its four parts, see [quad.Get\(d Direction\) Value](https://godoc.org/github.com/cayleygraph/quad#Quad.Get) 3. The term "Direction" comes about from the concept of traversing a graph. Take for example the triple `{A, follows, B}` and supposing you "select" the predicate `follows`. Now you want to traverse the graph, so you move in the `Object` direction, and you now have `B` selected. Whereas the high-level [path](glossary.md#path) abstraction for queries uses inbound/outbound predicates to represent movement on the graph, the bottom-level [iterator](glossary.md#iterator) mechanic uses Direction. ## path 1. Paths are just a set of helpers to build a query, but they are not that good for building something more complex. You can try using [Shapes](glossary.md#shape) for this - it will give you a full control of what the query actually does. [source](https://discourse.cayley.io/t/a-variety-of-questions/1183/2) 2. Path represents either a morphism \(a pre-defined path stored for later use\), or a concrete path, consisting of a morphism and an underlying QuadStore. [source](https://godoc.org/github.com/cayleygraph/cayley/query/path#Path) 3. Underlying code: ```go type Path struct { stack []morphism qs graph.QuadStore baseContext pathContext } type morphism struct { IsTag bool Reversal func(*pathContext) (morphism, *pathContext) Apply applyMorphism tags []string } type applyMorphism func(shape.Shape, *pathContext) (shape.Shape, *pathContext) ``` So, as previously stated, the [path](https://godoc.org/github.com/cayleygraph/cayley/query/path) package is just helper methods on top of the [shape](https://godoc.org/github.com/cayleygraph/cayley/graph/shape) package. ## morphism 1. Morphism is basically a path that is not attached to any particular quadstore or a particular starting point in the graph. Morphisms are meant to be used as a query part that can be applied to other queries to follow a path specified in the Morphism. A good example will be a `FollowRecursive` function that will apply a single morphism multiple times to get to all nodes that can be traversed recursively. [source](https://discourse.cayley.io/t/a-variety-of-questions/1183/2) ## iterator 1. So a graph query is roughly represented as a tree of iterators – things that implement graph.Iterator. An iterator is \(loosely\) a stand-in for a set of things that match a particular portion of the graph. [source](https://discourse.cayley.io/t/7-7-14-question-about-iterator/62) ## subiterator 1. So a graph query is roughly represented as a tree of iterators...Evaluation is merely calling Next\(\) repeatedly on the iterator at the top of the tree. Subiterators, then, are the branches and leaves of the tree. [source](https://discourse.cayley.io/t/7-7-14-question-about-iterator/62) 2. Example of converting the Cayley-Gremlin-Go-API query `g.V(“B”).In(“follows”).All()` into an iterator tree: * **HasA** \(subject\) – gets the things in the subject field for: * **And** – the intersection of: * **LinksTo \(predicate\)** links that have the predicate of…: * Fixed iterator containing “follows” – … just the node “follows”. * **LinksTo \(object\)** links that have the object field of: * Fixed iterator containing “B” – … just the node “B” ## LinkTo iterator 1. A LinksTo takes a subiterator of nodes, and contains an iteration of links which "link to" those nodes in a given direction. ... Can be seen as the dual of the HasA iterator. [source](https://github.com/cayleygraph/cayley/blob/1f53d04893ea9b2736e9b2277bbba3f47b88711a/graph/iterator/linksto.go#L17) * Next\(\)ing a LinksTo is straightforward -- iterate through all links to things in the subiterator, and then advance the subiterator, and do it again. * To restate in pseudo-code; `results` is what would be returned in successive `Next()` calls: ```go var results []quad.Quad for _, node := range linkTo.subIterator { for _, quad := range allQuads { if quad.Get(linkTo.direction) == node { results = append(results, quad) } } } ``` * Contains\(\)ing a LinksTo means, given a link, take the direction we care about and check if it's in our subiterator. * To restate in pseudo-code: ```go for _, node := range linkTo.subIterator { if theLink.Get(linkTo.direction) == node { return true } } return false ``` ## HasA iterator 1. The HasA takes a subiterator of links, and acts as an iterator of nodes in the given direction. The name comes from the idea that a "link HasA subject" or a "link HasA predicate". [source](https://github.com/cayleygraph/cayley/blob/41bf496d9dfe622b385c1482789480df8b106472/graph/iterator/hasa.go#L17) * Next\(\), [We have a subiterator we can get a value from, and we can take that resultant quad, pull our direction out of it, and return that.](https://github.com/cayleygraph/cayley/blob/41bf496d9dfe622b385c1482789480df8b106472/graph/iterator/hasa.go#L206) ```go var results []quad.Value for _, quad := range hasA.subIterator { results = append(results, quad.Get(hasA.direction)) } ``` * Contains\(\) ```go for _, quad := range hasA.subIterator { if quad.Get(hasA.direction) == theNode { return true } } return false ``` ## shape 1. Shape represent a query tree shape. [source](https://godoc.org/github.com/cayleygraph/cayley/graph/shape#Shape) ```go type Shape interface { BuildIterator(qs graph.QuadStore) graph.Iterator Optimize(ctx context.Context, r Optimizer) (Shape, bool) } ``` 2. This is the most interesting part of the query system - it describes how exactly the query looks like. ... This package also describes different query optimizations that are not specific to a backend. ... You can write a query using either Paths, Shapes or raw Iterators... [source](https://discourse.cayley.io/t/a-variety-of-questions/1183/2) 3. A Shape seems to be an abstract representation of a query, a level above Iterators and a level below Paths. You can perform various operations on it \(traverse inbound/outbound predicates, find unions and intersections, etc.\) and most importantly build a tree of Iterators from it, which will do the mechanical act of processing quads to find results. ## token 1. In the context of a [quad store](https://godoc.org/github.com/cayleygraph/cayley/graph#QuadStore), a [graph.Value](https://godoc.org/github.com/cayleygraph/cayley/graph#Value). However the backend wishes to implement it, a Value is merely a token to a quad or a node that the backing store itself understands, and the base iterators pass around. For example, in a very traditional, graphd-style graph, these are int64s \(guids of the primitives\). In a very direct sort of graph, these could be pointers to structs, or merely quads, or whatever works best for the backing store. ## reification 1. “With reification, we create a metagraph on top of our graph that represents the statement that we have here. We create a new node that represents a statement and points at the subject...” [source](https://neo4j.com/blog/rdf-triple-store-vs-labeled-property-graph-difference/) 2. Reifying a relationship means viewing it as an entity. The purpose of reifying a relationship is to make it explicit, when additional information needs to be added to it. Viewing a relationship as an entity, one can say that the entity reifies the relationship. This is called reification of a relationship. Like any other entity, it must be an instance of an entity type. [source](https://en.wikipedia.org/wiki/Reification_) ================================================ FILE: docs/graphql.md ================================================ # GraphQL Guide **Disclaimer:** Cayley's GraphQL implementation is not strictly a GraphQL, but only a query language with the same syntax and mostly the same rules. We will use [this simple dataset](https://github.com/cayleygraph/cayley/tree/87c9c341848b59924a054ebc2dd0f2bf8c57c6a9/data/testdata.nq) for our examples. Every query is represented by tree-like structure of nested objects and properties, similar to [MQL](mql.md). ```graphql { nodes{ id } } ``` This particular query is equivalent to all nodes in the graph, where `id` is the special field name for the value of the node itself. First root object in traditional GraphQL \(named `nodes` here\) represents a method that will be called on the server to get results. In our current implementation this name serves only as a placeholder and will always execute the object search query. Our example returns the following result: ```javascript { "data": { "nodes": [ {"id": "bob"}, {"id": "status"}, {"id": "cool_person"}, {"id": "alice"}, {"id": "greg"}, {"id": "emily"}, {"id": "smart_graph"}, {"id": "predicates"}, {"id": "dani"}, {"id": "fred"}, {"id": "smart_person"}, {"id": "charlie"}, {"id": "are"}, {"id": "follows"} ] } } ``` First level of JSON object corresponds to a request itself, thus either `data` field or `errors` will be present. Any nested objects will correspond to fields defined in query, including top-level name \(`nodes`\). ## Limit and pagination Maximal number of results can be limited using `first` keyword: ```graphql { nodes(first: 10){ id } } ``` Pagination can be done with `offset` keyword: ```graphql { nodes(offset: 5, first: 3){ id } } ``` This query returns objects 5-7. _Note: Values might be sorted differently, depending on what backend is used._ ## Properties Predicates \(or properties\) are added to the object to specify additional fields to load: ```graphql { nodes{ id, status } } ``` Results: ```javascript { "data": { "nodes": [ {"id": "bob", "status": "cool_person"}, {"id": "greg", "status": "cool_person"}, {"id": "dani", "status": "cool_person"}, {"id": "greg", "status": "smart_person"}, {"id": "emily", "status": "smart_person"} ] } } ``` All predicates are interpreted as IRIs and can be written in plain text or with angle brackets: `status` and `` are considered equal. Also, well-known namespaces like RDF, RDFS and Schema.org can be written in short form and will be expanded automatically: `schema:name` and `` will be expanded to ``. Properties are required to be present by default and can be set to optional with `@opt` or `@optional` directive: ```graphql { nodes{ id status @opt } } ``` Results: ```javascript { "data": { "nodes": [ {"id": "bob", "status": "cool_person"}, {"id": "status"}, {"id": "cool_person"}, {"id": "alice"}, {"id": "greg", "status": ["cool_person", "smart_person"]}, {"id": "emily", "status": "smart_person"}, {"id": "smart_graph"}, {"id": "predicates"}, {"id": "dani", "status": "cool_person"}, {"id": "fred"}, {"id": "smart_person"}, {"id": "charlie"}, {"id": "are"}, {"id": "follows"} ] } } ``` _Note: Since Cayley has no knowledge about property types and schema, it might decide to return a property as a single value for one object and as an array for another object. This behavior will be fixed in future versions._ ## Nested objects Objects and properties can be nested: ```graphql { nodes{ id follows { id } } } ``` All operations available on root also works for nested object, for example the limit: ```graphql { nodes(first: 10){ id follows(first: 1){ id } } } ``` ## Reversed predicates Any predicate can be reversed with `@rev` or `@reverse` directive \(search for "in" links instead of "out"\): ```graphql { nodes{ id followed: @rev { id } } } ``` ## Filters Objects can be filtered by specific values of properties: ```graphql { nodes(id: , status: "cool_person"){ id } } ``` Only exact match is supported for now. GraphQL names are interpreted as IRIs and string literals are interpreted as strings. Boolean, integer and float value are also supported and will be converted to `schema:Boolean`, `schema:Integer` and `schema:Float` accordingly. ## Labels Any fields and traversals can be filtered by quad label with `@label` directive: ```graphql { nodes{ id follows @label(v: ) { id, name follows @label { id, name } } } } ``` Label will be inherited by child objects. To reset label filter add `@label` directive without parameters. ## Expanding all properties To expand all properties of an object, `*` can be used instead of property name: ```graphql { nodes{ id follows {*} } } ``` ## Un-nest objects The following query will return objects with `{id: x, status: {name: y}}` structure: ```graphql { nodes{ id status { name } } } ``` It is possible to un-nest `status` field object into parent: ```graphql { nodes{ id status @unnest { status: name } } } ``` Resulted objects will have a flat structure: `{id: x, status: y}`. Arrays fields cannot be un-nested. You can still un-nest such fields by providing a limit directive \(will select the first value from array\): ```graphql { nodes{ id statuses(first: 1) @unnest { status: name } } } ``` ================================================ FILE: docs/gremlinapi.md ================================================ # GremlinAPI Cayley Gremlin API was renamed to [Gizmo](gizmoapi.md) to avoid confusion with TinkerPop Gremlin API. ================================================ FILE: docs/hacking.md ================================================ # HACKING See [Contributing.md](contributing.md) ================================================ FILE: docs/http.md ================================================ # HTTP Methods This file covers deprecated v1 HTTP API. All the methods of v2 HTTP API is described in OpenAPI/Swagger [spec](https://github.com/cayleygraph/cayley/tree/87c9c341848b59924a054ebc2dd0f2bf8c57c6a9/docs/api/swagger.yml) and can be viewed by importing `https://raw.githubusercontent.com/cayleygraph/cayley/master/docs/api/swagger.yml` URL into [Swagger Editor](https://editor.swagger.io/) or [Swagger UI demo](http://petstore.swagger.io/). ## Gephi Cayley supports streaming to Gephi via [GraphStream](gephigraphstream.md). ## API v1 Unless otherwise noted, all URIs take a POST command. ### Queries and Results #### `/api/v1/query/gizmo` POST Body: Javascript source code of the query Response: JSON results, depending on the query. #### `/api/v1/query/graphql` POST Body: [GraphQL](graphql.md) query Response: JSON results, depending on the query. #### `/api/v1/query/mql` POST Body: JSON MQL query Response: JSON results, with a query wrapper: ```javascript { "result": } ``` If the JSON is invalid or an error occurs: ```javascript { "error": "Error message" } ``` ### Query Shapes Result form: ```javascript { "nodes": [{ "id" : integer, "tags": ["list of tags from the query"], "values": ["known values from the query"], "is_link_node": bool, // Does the node represent the link or the node (the oval shapes) "is_fixed": bool // Is the node a fixed starting point of the query }], "links": [{ "source": integer, // Node ID "target": integer, // Node ID "link_node": integer // Node ID }] } ``` #### `/api/v1/shape/gizmo` POST Body: Javascript source code of the query Response: JSON description of the last query executed. #### `/api/v1/shape/mql` POST Body: JSON MQL query Response: JSON description of the query. ### Write commands Responses come in the form 200 Success: ```javascript { "result": "Success message." } ``` 400 / 500 Error: ```javascript { "error": "Error message." } ``` #### `/api/v1/write` POST Body: JSON quads ```javascript [{ "subject": "Subject Node", "predicate": "Predicate Node", "object": "Object node", "label": "Label node" // Optional }] // More than one quad allowed. ``` Response: JSON response message #### `/api/v1/write/file/nquad` POST Body: Form-encoded body: * Key: `NQuadFile`, Value: N-Quad file to write. Response: JSON response message Example: ```text curl http://localhost:64210/api/v1/write/file/nquad -F NQuadFile=@30k.n3 ``` #### `/api/v1/delete` POST Body: JSON quads ```javascript [{ "subject": "Subject Node", "predicate": "Predicate Node", "object": "Object node", "label": "Label node" // Optional }] // More than one quad allowed. ``` Response: JSON response message. ================================================ FILE: docs/installation.md ================================================ # Install Cayley ## Install Cayley on Ubuntu ```text snap install --edge --devmode cayley ``` ## Install Cayley on macOS ### Install Homebrew macOS does not include the Homebrew brew package by default. Install brew using the [official instructions](https://brew.sh/#install) ### Install Cayley ```bash brew install cayley ``` ## Install Cayley with Docker ```bash docker run -p 64210:64210 cayleygraph/cayley ``` For more information see [Container Documentation](deployment/container.md) ## Build from Source See instructions in [Contributing](getting-involved/contributing.md) ================================================ FILE: docs/k8s/README.md ================================================ # k8s ================================================ FILE: docs/k8s/cayley-mongo.yml ================================================ kind: Namespace apiVersion: v1 metadata: name: cayley --- apiVersion: v1 kind: Service metadata: name: mongo namespace: cayley labels: name: mongo spec: ports: - name: mgo port: 27017 targetPort: mgo clusterIP: None selector: role: mongo --- apiVersion: apps/v1beta1 kind: StatefulSet metadata: name: mongo namespace: cayley spec: serviceName: "mongo" replicas: 3 template: metadata: namespace: cayley labels: role: mongo environment: test spec: terminationGracePeriodSeconds: 10 containers: - name: mongo image: mongo:3 command: - mongod - "--replSet" - rs0 - "--smallfiles" - "--noprealloc" ports: - name: mgo containerPort: 27017 volumeMounts: - name: mongo-pvc mountPath: /data/db - name: mongo-sidecar image: cvallance/mongo-k8s-sidecar env: - name: MONGO_SIDECAR_POD_LABELS value: "role=mongo,environment=test" volumeClaimTemplates: - metadata: name: mongo-pvc spec: accessModes: [ "ReadWriteOnce" ] storageClassName: standard resources: requests: storage: 20Gi --- kind: Service apiVersion: v1 metadata: name: cayley namespace: cayley spec: selector: app: cayley ports: - protocol: TCP port: 80 targetPort: http --- apiVersion: extensions/v1beta1 kind: Deployment metadata: name: cayley namespace: cayley spec: replicas: 1 template: metadata: namespace: cayley labels: app: cayley spec: initContainers: - name: cayley-init image: ghcr.io/cayleygraph/cayley:v0.7.8 command: - cayley - init - -d=mongo - -a=mongo containers: - name: cayley image: ghcr.io/cayleygraph/cayley:v0.7.8 command: - cayley - http - --init # TODO: remove once initContainers works properly - -d=mongo - -a=mongo - --host=:64210 ports: - name: http containerPort: 64210 ================================================ FILE: docs/k8s/cayley-single.yml ================================================ kind: Namespace apiVersion: v1 metadata: name: cayley --- kind: Service apiVersion: v1 metadata: name: cayley namespace: cayley spec: selector: app: cayley ports: - protocol: TCP port: 80 targetPort: http --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: cayley-pvc namespace: cayley spec: accessModes: - ReadWriteOnce resources: requests: storage: 20Gi storageClassName: standard --- apiVersion: extensions/v1beta1 kind: Deployment metadata: name: cayley namespace: cayley spec: replicas: 1 # cannot be really scaled because of local backend template: metadata: namespace: cayley labels: app: cayley spec: initContainers: - name: cayley-init image: ghcr.io/cayleygraph/cayley:v0.7.8 command: - cayley - init - -c=/etc/cayley.json volumeMounts: - mountPath: /data name: database containers: - name: cayley image: ghcr.io/cayleygraph/cayley:v0.7.8 command: - cayley - http - --init # TODO: remove once initContainers is out of beta - -c=/etc/cayley.json - --host=:64210 ports: - name: http containerPort: 64210 volumeMounts: - mountPath: /data name: database volumes: - name: database persistentVolumeClaim: claimName: cayley-pvc ================================================ FILE: docs/k8s/k8s.md ================================================ # Running in Kubernetes Most examples requires Kubernetes 1.5+ and PersistentVolumes are configured. After running scripts namespace `cayley` will be created and service with the same name will be available in cluster. Service is of type `ClusterIP` by default. If you want to expose it, consider changing type to `LoadBalancer`. ## Single instance \(Bolt\) This is a simplest possible configuration: single Cayley instance with persistent storage, using Bolt as a backend. ```bash kubectl create -f ./docs/k8s/cayley-single.yml ``` ## Distributed \(MongoDB cluster\) This example is based on [thesandlord/mongo-k8s-sidecar](https://github.com/thesandlord/mongo-k8s-sidecar) and runs Cayley on top of 3-node Mongo cluster. ```bash kubectl create -f ./docs/k8s/cayley-mongo.yml ``` ## Distributed _TODO: PostgreSQL, CockroachDB_ ================================================ FILE: docs/locations.md ================================================ # Locations of parts of Cayley ## Community * Where is the beating heart of this community? [https://discourse.cayley.io/](https://discourse.cayley.io/) * Where is the mailing list? [https://discourse.cayley.io/](https://discourse.cayley.io/) \(enable mailing list mode under options\) * Where is the chat room? [cayleygraph.slack.com](https://cayleygraph.slack.com) -- Invite [here](https://cayley-slackin.herokuapp.com/) * Where should I start contributing? [Contributing.md](contributing.md) \(Also: [How to get involved!](https://discourse.cayley.io/t/how-to-get-involved/44)\) * Where is the site-content? [https://github.com/cayleygraph/site-content](https://github.com/cayleygraph/site-content) * Where are issues? [https://github.com/cayleygraph/cayley/issues](https://github.com/cayleygraph/cayley/issues) * Where is the graph source code? [https://github.com/cayleygraph/cayley](https://github.com/cayleygraph/cayley) * Where are pull requests? [https://github.com/cayleygraph/cayley/pulls](https://github.com/cayleygraph/cayley/pulls) * Where is the wiki? It was at [https://github.com/cayleygraph/cayley/wiki](https://github.com/cayleygraph/cayley/wiki), but we are deprecating it. * Where is the \(old\) mailing list? It was at [https://groups.google.com/forum/\#!forum/cayley-users](https://groups.google.com/forum/#!forum/cayley-users), but we are deprecating it for this forum \(and its mailing list mode\). ================================================ FILE: docs/migration.md ================================================ # Migration ## From Cayley 0.6.1 to 0.7.x First you need to dump all the data from the database via v0.6.1: ```bash ./cayley-0.6 dump --db --dbpath
--dump_type pquads --dump ./data.pq.gz ``` or using config file: ```bash ./cayley-0.6 dump --config --dump_type pquads --dump ./data.pq.gz ``` And load the data into new database via v0.7.x \(`pq` file extension is important\): ```bash ./cayley load --init -d -a -i ./data.pq.gz ``` or using config file: ```bash ./cayley load --init -c -i ./data.pq.gz ``` ### Dump via text format An above guide uses Cayley-specific binary format to avoid encoding and parsing overhead and to compress output file better. As an alternative, a standard nquads file format can be used to dump and load data \(note `nq` extension\): ```bash ./cayley-0.6 dump --config --dump ./data.nq.gz ./cayley load --init -c -i ./data.nq.gz ``` ### Bolt, LevelDB, MongoDB Cayley v0.7.0 is still able to read and write databases of these types in old format. It can be accessed by changing backend name from `bolt`/`leveldb`/`mongo` to `bolt1`/`leveldb1`/`mongo1`. Thus, you can use guide for moving data between different backends for v0.7.x \(see below\). **Note:** support for old versions will be dropped starting from v0.7.1. ### SQL Data format for SQL between v0.6.1 and v0.7.x has not changed significantly. Database can be upgraded by executing the following SQL statements: ```sql ALTER TABLE quads DROP id; ALTER TABLE nodes ADD refs INT DEFAULT 0 NOT NULL; ALTER TABLE nodes ALTER COLUMN refs DROP DEFAULT; UPDATE nodes SET refs = (SELECT count(1) FROM quads WHERE subject_hash = nodes.hash) + (SELECT count(1) FROM quads WHERE predicate_hash = nodes.hash) + (SELECT count(1) FROM quads WHERE object_hash = nodes.hash) + (SELECT count(1) FROM quads WHERE label_hash = nodes.hash); ``` ## From different backend \(Cayley 0.7+\) First you need to dump all the data from old backend \(`pq` extension is important\): ```bash ./cayley dump -d -a
-o ./data.pq.gz ``` or using config file: ```bash ./cayley dump -c -o ./data.pq.gz ``` And load the data into a new backend and/or database: ```bash ./cayley load --init -d -a -i ./data.pq.gz ``` or using config file: ```bash ./cayley load --init -c -i ./data.pq.gz ``` ### Dump via text format An above guide uses Cayley-specific binary format to avoid encoding and parsing overhead and to compress output file better. As an alternative, a standard nquads file format can be used to dump and load data \(note `nq` extension\): ```bash ./cayley dump -c -o ./data.nq.gz ./cayley load --init -c -i ./data.nq.gz ``` ================================================ FILE: docs/mql.md ================================================ # MQL Guide ## General Cayley's MQL implementation is a work-in-progress clone of [Freebase's MQL API](https://developers.google.com/freebase/mql/). At the moment, it supports very basic queries without some of the extended features. It also aims to be database-agnostic, meaning that the schema inference from Freebase does not \(yet\) apply. Every JSON Object can be thought of as a node in the graph, and wrapping an object in a list means there may be several of these, or it may be repeated. A simple query like: ```javascript [{ "id": null }] ``` Is equivalent to all nodes in the graph, where "id" is the special keyword for the value of the node. Predicates are added to the object to specify constraints. ```javascript [{ "id": null, "some_predicate": "some value" }] ``` Predicates can take as values objects or lists of objects \(subqueries\), strings and numbers \(literal IDs that must match -- equivalent to the object {"id": "value"}\) or null, which indicates that, while the object must have a predicate that matches, the matching values will replace the null. A single null is one such value, an empty list will be filled with all such values, as strings. ## Keywords * `id`: The value of the node. ## Reverse Predicates Predicates always assume a forward direction. That is, ```javascript [{ "id": "A", "some_predicate": "B" }] ``` will only match if the quad ```text A some_predicate B . ``` exists. In order to reverse the directions, "!predicates" are used. So that: ```javascript [{ "id": "A", "!some_predicate": "B" }] ``` will only match if the quad ```text B some_predicate A . ``` exists. ## Multiple Predicates JSON does not specify the behavior of objects with the same key. In order to have separate constraints for the same predicate, the prefix "@name:" can be applied to any predicate. This is slightly different from traditional MQL in that fully-qualified http paths may be common predicates, so we have an "@name:" prefix instead. ```javascript [{ "id": "A", "@x:some_predicate": "B", "@y:some_predicate": "C" }] ``` Will only match if _both_ ```text A some_predicate B . A some_predicate C . ``` exist. This combines with the reversal rule to create paths like `"@a:!some_predicate"` ================================================ FILE: docs/query-languages/gephigraphstream.md ================================================ # Gephi GraphStream Cayley supports graph visualisation in Gephi using GraphStream API. Enpoint can be accessed by adding URL `http://localhost:64210/gephi/gs` to Gephi GraphStream client. ## Options ### `limit` Default: `10000` Sets a maximal number of object that will be streamed. Depending on stream mode this could be either nodes or quads. Values less than 0 interpreted as "no limit". ### `mode` Sets streaming mode. Supported values: * `raw` \(default\) - all or selected quads * `nodes` - nodes with properties #### Raw mode In this mode Cayley directly streams selected quads to Gephi. Example URLs: `/gephi/gs?mode=raw&pred=&limit=-1` \(all quads\) `/gephi/gs?mode=raw&sub=&pred=,&limit=-1` \(links from `` via either `` or ``\) Parameters: * `limit` - maximal number of quads returned * `sub`,`pred`,`obj`,`label` - only show quads with specified values of Subject, Predicate, Object or Label This mode may be useful to visualize small subgraphs, or graphs without metadata such as types and properties. In case of later, large number of quads will be pointing to nodes describing common types or property names. For this kind of graphs `nodes` mode should be used. #### Nodes with properties Example URL: `/gephi/gs?mode=nodes&limit=-1` In this mode Cayley streams all nodes and links associated with them, but instead of streaming common quads such as types it will inline them as Gephi properties. Limit corresponds to a number of nodes returned. By default, all predicate will be streamed as in `raw` mode, except well-known predicates and ones with ` = "true"`. List of well-known predicates includes: * `` \(``\) * `` \(``\) * `` \(``\) To add custom predicates, write a special triple to database: ```text "true"^^ . ``` This allows to partition nodes by type or specific property values. Note: Only one value per predicate is supported for inlined properties. By default nodes will have random positions. To specify an exact position for specific node add `` and `` properties: ```text "10"^^ "-12.3"^^ ``` ================================================ FILE: docs/query-languages/gizmoapi.md ================================================ # Gizmo API ![Autogenerated file](https://img.shields.io/badge/file-generated-orange.svg) ## The `graph` object Name: `graph`, Alias: `g` This is the only special object in the environment, generates the query objects. Under the hood, they're simple objects that get compiled to a Go iterator tree when executed. ### `graph.addDefaultNamespaces()` AddDefaultNamespaces register all default namespaces for automatic IRI resolution. ### `graph.addNamespace(pref, ns)` AddNamespace associates prefix with a given IRI namespace. ### `graph.emit(*)` Emit adds data programmatically to the JSON result list. Can be any JSON type. ```javascript g.emit({ name: "bob" }); // push {"name":"bob"} as a result ``` ### `graph.loadNamespaces()` LoadNamespaces loads all namespaces saved to graph. ### `graph.M()` M is a shorthand for Morphism. ### `graph.Morphism()` Morphism creates a morphism path object. Unqueryable on it's own, defines one end of the path. Saving these to variables with ```javascript var shorterPath = graph .Morphism() .out("foo") .out("bar"); ``` is the common use case. See also: path.follow\(\), path.followR\(\). ### `graph.IRI(s)` Uri creates an IRI values from a given string. ### `graph.V(*)` V is a shorthand for Vertex. ### `graph.Vertex([nodeId],[nodeId]...)` Vertex starts a query path at the given vertex/vertices. No ids means "all vertices". Arguments: * `nodeId` \(Optional\): A string or list of strings representing the starting vertices. Returns: Path object ## Path object Both `.Morphism()` and `.Vertex()` create path objects, which provide the following traversal methods. Note that `.Vertex()` returns a query object, which is a subclass of path object. For these examples, suppose we have the following graph: ```text +-------+ +------+ | alice |----- ->| fred |<-- +-------+ \---->+-------+-/ +------+ \-+-------+ ----->| #bob# | | |*emily*| +---------+--/ --->+-------+ | +-------+ | charlie | / v +---------+ / +--------+ \--- +--------+ |*#greg#*| \-->| #dani# |------------>+--------+ +--------+ ``` Where every link is a `` relationship, and the nodes with an extra `#` in the name have an extra `` link. As in, ```text -- --> "cool_person" ``` Perhaps these are the influencers in our community. So too are extra `*`s in the name -- these are our smart people, according to the `` label, eg, the quad: ```text "smart_person" . ``` ### `path.all()` All executes the query and adds the results, with all tags, as a string-to-string \(tag to node\) map in the output set, one for each path that a traversal could take. ### `path.and(path)` And is an alias for Intersect. ### `path.as(tags)` As is an alias for Tag. ### `path.back([tag])` Back returns current path to a set of nodes on a given tag, preserving all constraints. If still valid, a path will now consider their vertex to be the same one as the previously tagged one, with the added constraint that it was valid all the way here. Useful for traversing back in queries and taking another route for things that have matched so far. Arguments: * `tag`: A previous tag in the query to jump back to. Example: ```javascript // Start from all nodes, save them into start, follow any status links, // jump back to the starting node, and find who follows them. Return the result. // Results are: // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""}, // {"id": "", "start": ""} g.V() .Tag("start") .out("") .back("start") .in("") .all(); ``` ### `path.both([predicatePath], [tags])` Both follow the predicate in either direction. Same as Out or In. Example: ```javascript // Find all followers/followees of fred. Returns bob, emily and greg g.V("") .both("") .all(); ``` ### `path.count()` Count returns a number of results and returns it as a value. Example: ```javascript // Save count as a variable var n = g.V().count(); // Send it as a query result g.emit(n); ``` ### `path.difference(path)` Difference is an alias for Except. ### `path.except(path)` Except removes all paths which match query from current path. In a set-theoretic sense, this is \(A - B\). While `g.V().except(path)` to achieve `U - B = !B` is supported, it's often very slow. Example: ```javascript var cFollows = g.V("").out(""); var dFollows = g.V("").out(""); // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob. cFollows.except(dFollows).all(); // The set (dani) -- what charlie follows that dani does not also follow. // Equivalently, g.V("").out("").except(g.V("").out("")).all() ``` ### `path.filter(args)` Filter applies constraints to a set of nodes. Can be used to filter values by range or match strings. #### `path.filter(regex(expression, includeIRIs))` Filters by match a regular expression \([syntax](https://github.com/google/re2/wiki/Syntax)\). By default works only on literals unless includeEntities is set to `true`. ### `path.follow(path)` Follow is the way to use a path prepared with Morphism. Applies the path chain on the morphism object to the current path. Starts as if at the g.M\(\) and follows through the morphism path. Example: ```javascript var friendOfFriend = g .Morphism() .out("") .out(""); // Returns the followed people of who charlie follows -- a simplistic "friend of my friend" // and whether or not they have a "cool" status. Potential for recommending followers abounds. // Returns bob and greg g.V("") .follow(friendOfFriend) .has("", "cool_person") .all(); ``` ### `path.followR(path)` FollowR is the same as Follow but follows the chain in the reverse direction. Flips "In" and "Out" where appropriate, the net result being a virtual predicate followed in the reverse direction. Starts at the end of the morphism and follows it backwards \(with appropriate flipped directions\) to the g.M\(\) location. Example: ```javascript var friendOfFriend = g .Morphism() .out("") .out(""); // Returns the third-tier of influencers -- people who follow people who follow the cool people. // Returns charlie (from bob), charlie (from greg), bob and emily g.V() .has("", "cool_person") .followR(friendOfFriend) .all(); ``` ### `path.followRecursive(*)` FollowRecursive is the same as Follow but follows the chain recursively. Starts as if at the g.M\(\) and follows through the morphism path multiple times, returning all nodes encountered. Example: ```javascript var friend = g.Morphism().out(""); // Returns all people in Charlie's network. // Returns bob and dani (from charlie), fred (from bob) and greg (from dani). g.V("") .followRecursive(friend) .all(); ``` ### `path.forEach(callback) or (limit, callback)` ForEach calls callback\(data\) for each result, where data is the tag-to-string map as in All case. Arguments: * `limit` \(Optional\): An integer value on the first `limit` paths to process. * `callback`: A javascript function of the form `function(data)` Example: ```javascript // Simulate query.all().all() graph.V("").ForEach(function(d) { g.emit(d); }); ``` ### `path.getLimit(limit)` GetLimit is the same as All, but limited to the first N unique nodes at the end of the path, and each of their possible traversals. ### `path.has(predicate, object)` Has filters all paths which are, at this point, on the subject for the given predicate and object, but do not follow the path, merely filter the possible paths. Usually useful for starting with all nodes, or limiting to a subset depending on some predicate/value pair. Arguments: * `predicate`: A string for a predicate node. * `object`: A string for a object node or a set of filters to find it. Example: ```javascript // Start from all nodes that follow bob -- results in alice, charlie and dani g.V() .has("", "") .all(); // People charlie follows who then follow fred. Results in bob. g.V("") .out("") .has("", "") .all(); // People with friends who have names sorting lower then "f". g.V() .has("", gt("")) .all(); ``` ### `path.hasR(*)` HasR is the same as Has, but sets constraint in reverse direction. ### `path.in([predicatePath], [tags])` In is inverse of Out. Starting with the nodes in `path` on the object, follow the quads with predicates defined by `predicatePath` to their subjects. Arguments: * `predicatePath` \(Optional\): One of: * null or undefined: All predicates pointing into this node * a string: The predicate name to follow into this node * a list of strings: The predicates to follow into this node * a query path object: The target of which is a set of predicates to follow. * `tags` \(Optional\): One of: * null or undefined: No tags * a string: A single tag to add the predicate used to the output set. * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. Example: ```javascript // Find the cool people, bob, dani and greg g.V("cool_person") .in("") .all(); // Find who follows bob, in this case, alice, charlie, and dani g.V("") .in("") .all(); // Find who follows the people emily follows, namely, bob and emily g.V("") .out("") .in("") .all(); ``` ### `path.inPredicates()` InPredicates gets the list of predicates that are pointing in to a node. Example: ```javascript // bob only has "" predicates pointing inward // returns "" g.V("") .inPredicates() .all(); ``` ### `path.intersect(path)` Intersect filters all paths by the result of another query path. This is essentially a join where, at the stage of each path, a node is shared. Example: ```javascript var cFollows = g.V("").out(""); var dFollows = g.V("").out(""); // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob. cFollows.intersect(dFollows).all(); // Equivalently, g.V("").out("").And(g.V("").out("")).all() ``` ### `path.is(node, [node..])` Filter all paths to ones which, at this point, are on the given node. Arguments: * `node`: A string for a node. Can be repeated or a list of strings. Example: ```javascript // Starting from all nodes in the graph, find the paths that follow bob. // Results in three paths for bob (from alice, charlie and dani).all() g.V() .out("") .is("") .all(); ``` ### `path.labelContext([labelPath], [tags])` LabelContext sets \(or removes\) the subgraph context to consider in the following traversals. Affects all In\(\), Out\(\), and Both\(\) calls that follow it. The default LabelContext is null \(all subgraphs\). Arguments: * `predicatePath` \(Optional\): One of: * null or undefined: In future traversals, consider all edges, regardless of subgraph. * a string: The name of the subgraph to restrict traversals to. * a list of strings: A set of subgraphs to restrict traversals to. * a query path object: The target of which is a set of subgraphs. * `tags` \(Optional\): One of: * null or undefined: No tags * a string: A single tag to add the last traversed label to the output set. * a list of strings: Multiple tags to use as keys to save the label used to the output set. Example: ```javascript // Find the status of people Dani follows g.V("") .out("") .out("") .all(); // Find only the statuses provided by the smart_graph g.V("") .out("") .labelContext("") .out("") .all(); // Find all people followed by people with statuses in the smart_graph. g.V() .labelContext("") .in("") .labelContext(null) .in("") .all(); ``` ### `path.labels()` Labels gets the list of inbound and outbound quad labels ### `path.limit(limit)` Limit limits a number of nodes for current path. Arguments: * `limit`: A number of nodes to limit results to. Example: ```javascript // Start from all nodes that follow bob, and limit them to 2 nodes -- results in alice and charlie g.V() .has("", "") .limit(2) .all(); ``` ### `path.map(*)` Map is a alias for ForEach. ### `path.or(path)` Or is an alias for Union. ### `path.out([predicatePath], [tags])` Out is the work-a-day way to get between nodes, in the forward direction. Starting with the nodes in `path` on the subject, follow the quads with predicates defined by `predicatePath` to their objects. Arguments: * `predicatePath` \(Optional\): One of: * null or undefined: All predicates pointing out from this node * a string: The predicate name to follow out from this node * a list of strings: The predicates to follow out from this node * a query path object: The target of which is a set of predicates to follow. * `tags` \(Optional\): One of: * null or undefined: No tags * a string: A single tag to add the predicate used to the output set. * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. Example: ```javascript // The working set of this is bob and dani g.V("") .out("") .all(); // The working set of this is fred, as alice follows bob and bob follows fred. g.V("") .out("") .out("") .all(); // Finds all things dani points at. Result is bob, greg and cool_person g.V("") .out() .all(); // Finds all things dani points at on the status linkage. // Result is bob, greg and cool_person g.V("") .out(["", ""]) .all(); // Finds all things dani points at on the status linkage, given from a separate query path. // Result is {"id": "cool_person", "pred": ""} g.V("") .out(g.V(""), "pred") .all(); ``` ### `path.outPredicates()` OutPredicates gets the list of predicates that are pointing out from a node. Example: ```javascript // bob has "" and "" edges pointing outwards // returns "", "" g.V("") .outPredicates() .all(); ``` ### `path.save(predicate, tag)` Save saves the object of all quads with predicate into tag, without traversal. Arguments: * `predicate`: A string for a predicate node. * `tag`: A string for a tag key to store the object node. Example: ```javascript // Start from dani and bob and save who they follow into "target" // Returns: // {"id" : "", "target": "" }, // {"id" : "", "target": "" }, // {"id" : "", "target": "" } g.V("", "") .save("", "target") .all(); ``` ### `path.saveInPredicates(tag)` SaveInPredicates tags the list of predicates that are pointing in to a node. Example: ```javascript // bob only has "" predicates pointing inward // returns {"id":"", "pred":""} g.V("") .saveInPredicates("pred") .all(); ``` ### `path.saveOpt(*)` SaveOpt is the same as Save, but returns empty tags if predicate does not exists. ### `path.saveOptR(*)` SaveOptR is the same as SaveOpt, but tags values via reverse predicate. ### `path.saveOutPredicates(tag)` SaveOutPredicates tags the list of predicates that are pointing out from a node. Example: ```javascript // bob has "" and "" edges pointing outwards // returns {"id":"", "pred":""} g.V("") .saveInPredicates("pred") .all(); ``` ### `path.saveR(*)` SaveR is the same as Save, but tags values via reverse predicate. ### `path.skip(offset)` Skip skips a number of nodes for current path. Arguments: * `offset`: A number of nodes to skip. Example: ```javascript // Start from all nodes that follow bob, and skip 2 nodes -- results in dani g.V() .has("", "") .skip(2) .all(); ``` ### `path.tag(tags)` Tag saves a list of nodes to a given tag. In order to save your work or learn more about how a path got to the end, we have tags. The simplest thing to do is to add a tag anywhere you'd like to put each node in the result set. Arguments: * `tag`: A string or list of strings to act as a result key. The value for tag was the vertex the path was on at the time it reached "Tag" Example: ```javascript // Start from all nodes, save them into start, follow any status links, and return the result. // Results are: // {"id": "cool_person", "start": ""}, // {"id": "cool_person", "start": ""}, // {"id": "cool_person", "start": ""}, // {"id": "smart_person", "start": ""}, // {"id": "smart_person", "start": ""} g.V() .tag("start") .out("") .all(); ``` ### `path.tagArray(*)` TagArray is the same as ToArray, but instead of a list of top-level nodes, returns an Array of tag-to-string dictionaries, much as All would, except inside the JS environment. Example: ```javascript // bobTags contains an Array of followers of bob (alice, charlie, dani). var bobTags = g .V("") .tag("name") .in("") .tagArray(); // nameValue should be the string "" var nameValue = bobTags[0]["name"]; ``` ### `path.tagValue()` TagValue is the same as TagArray, but limited to one result node. Returns a tag-to-string map. ### `path.toArray(*)` ToArray executes a query and returns the results at the end of the query path as an JS array. Example: ```javascript // bobFollowers contains an Array of followers of bob (alice, charlie, dani). var bobFollowers = g .V("") .in("") .toArray(); ``` ### `path.toValue()` ToValue is the same as ToArray, but limited to one result node. ### `path.union(path)` Union returns the combined paths of the two queries. Notice that it's per-path, not per-node. Once again, if multiple paths reach the same destination, they might have had different ways of getting there \(and different tags\). See also: `path.Tag()` Example: ```javascript var cFollows = g.V("").out(""); var dFollows = g.V("").out(""); // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob (from charlie), dani, bob (from dani), and greg. cFollows.union(dFollows).all(); ``` ### `path.unique()` Unique removes duplicate values from the path. ### `path.order()` Order returns values from the path in ascending order. ================================================ FILE: docs/query-languages/graphql.md ================================================ # GraphQL Guide **Disclaimer:** Cayley's GraphQL implementation is not strictly a GraphQL, but only a query language with the same syntax and mostly the same rules. We will use [this simple dataset](https://github.com/cayleygraph/cayley/tree/87c9c341848b59924a054ebc2dd0f2bf8c57c6a9/data/testdata.nq) for our examples. Every query is represented by tree-like structure of nested objects and properties, similar to [MQL](mql.md). ```graphql { nodes{ id } } ``` This particular query is equivalent to all nodes in the graph, where `id` is the special field name for the value of the node itself. First root object in traditional GraphQL \(named `nodes` here\) represents a method that will be called on the server to get results. In our current implementation this name serves only as a placeholder and will always execute the object search query. Our example returns the following result: ```javascript { "data": { "nodes": [ {"id": "bob"}, {"id": "status"}, {"id": "cool_person"}, {"id": "alice"}, {"id": "greg"}, {"id": "emily"}, {"id": "smart_graph"}, {"id": "predicates"}, {"id": "dani"}, {"id": "fred"}, {"id": "smart_person"}, {"id": "charlie"}, {"id": "are"}, {"id": "follows"} ] } } ``` First level of JSON object corresponds to a request itself, thus either `data` field or `errors` will be present. Any nested objects will correspond to fields defined in query, including top-level name \(`nodes`\). ## Limit and pagination Maximal number of results can be limited using `first` keyword: ```graphql { nodes(first: 10){ id } } ``` Pagination can be done with `offset` keyword: ```graphql { nodes(offset: 5, first: 3){ id } } ``` This query returns objects 5-7. _Note: Values might be sorted differently, depending on what backend is used._ ## Properties Predicates \(or properties\) are added to the object to specify additional fields to load: ```graphql { nodes{ id, status } } ``` Results: ```javascript { "data": { "nodes": [ {"id": "bob", "status": "cool_person"}, {"id": "greg", "status": "cool_person"}, {"id": "dani", "status": "cool_person"}, {"id": "greg", "status": "smart_person"}, {"id": "emily", "status": "smart_person"} ] } } ``` All predicates are interpreted as IRIs and can be written in plain text or with angle brackets: `status` and `` are considered equal. Also, well-known namespaces like RDF, RDFS and Schema.org can be written in short form and will be expanded automatically: `schema:name` and `` will be expanded to ``. Properties are required to be present by default and can be set to optional with `@opt` or `@optional` directive: ```graphql { nodes{ id status @opt } } ``` Results: ```javascript { "data": { "nodes": [ {"id": "bob", "status": "cool_person"}, {"id": "status"}, {"id": "cool_person"}, {"id": "alice"}, {"id": "greg", "status": ["cool_person", "smart_person"]}, {"id": "emily", "status": "smart_person"}, {"id": "smart_graph"}, {"id": "predicates"}, {"id": "dani", "status": "cool_person"}, {"id": "fred"}, {"id": "smart_person"}, {"id": "charlie"}, {"id": "are"}, {"id": "follows"} ] } } ``` _Note: Since Cayley has no knowledge about property types and schema, it might decide to return a property as a single value for one object and as an array for another object. This behavior will be fixed in future versions._ ## Nested objects Objects and properties can be nested: ```graphql { nodes{ id follows { id } } } ``` All operations available on root also works for nested object, for example the limit: ```graphql { nodes(first: 10){ id follows(first: 1){ id } } } ``` ## Reversed predicates Any predicate can be reversed with `@rev` or `@reverse` directive \(search for "in" links instead of "out"\): ```graphql { nodes{ id followed: @rev { id } } } ``` ## Filters Objects can be filtered by specific values of properties: ```graphql { nodes(id: , status: "cool_person"){ id } } ``` Only exact match is supported for now. GraphQL names are interpreted as IRIs and string literals are interpreted as strings. Boolean, integer and float value are also supported and will be converted to `schema:Boolean`, `schema:Integer` and `schema:Float` accordingly. ## Labels Any fields and traversals can be filtered by quad label with `@label` directive: ```graphql { nodes{ id follows @label(v: ) { id, name follows @label { id, name } } } } ``` Label will be inherited by child objects. To reset label filter add `@label` directive without parameters. ## Expanding all properties To expand all properties of an object, `*` can be used instead of property name: ```graphql { nodes{ id follows {*} } } ``` ## Un-nest objects The following query will return objects with `{id: x, status: {name: y}}` structure: ```graphql { nodes{ id status { name } } } ``` It is possible to un-nest `status` field object into parent: ```graphql { nodes{ id status @unnest { status: name } } } ``` Resulted objects will have a flat structure: `{id: x, status: y}`. Arrays fields cannot be un-nested. You can still un-nest such fields by providing a limit directive \(will select the first value from array\): ```graphql { nodes{ id statuses(first: 1) @unnest { status: name } } } ``` ================================================ FILE: docs/query-languages/mql.md ================================================ # MQL Guide ## General Cayley's MQL implementation is a work-in-progress clone of [Freebase's MQL API](https://developers.google.com/freebase/mql/). At the moment, it supports very basic queries without some of the extended features. It also aims to be database-agnostic, meaning that the schema inference from Freebase does not \(yet\) apply. Every JSON Object can be thought of as a node in the graph, and wrapping an object in a list means there may be several of these, or it may be repeated. A simple query like: ```javascript [{ "id": null }] ``` Is equivalent to all nodes in the graph, where "id" is the special keyword for the value of the node. Predicates are added to the object to specify constraints. ```javascript [{ "id": null, "some_predicate": "some value" }] ``` Predicates can take as values objects or lists of objects \(subqueries\), strings and numbers \(literal IDs that must match -- equivalent to the object {"id": "value"}\) or null, which indicates that, while the object must have a predicate that matches, the matching values will replace the null. A single null is one such value, an empty list will be filled with all such values, as strings. ## Keywords * `id`: The value of the node. ## Reverse Predicates Predicates always assume a forward direction. That is, ```javascript [{ "id": "A", "some_predicate": "B" }] ``` will only match if the quad ```text A some_predicate B . ``` exists. In order to reverse the directions, "!predicates" are used. So that: ```javascript [{ "id": "A", "!some_predicate": "B" }] ``` will only match if the quad ```text B some_predicate A . ``` exists. ## Multiple Predicates JSON does not specify the behavior of objects with the same key. In order to have separate constraints for the same predicate, the prefix "@name:" can be applied to any predicate. This is slightly different from traditional MQL in that fully-qualified http paths may be common predicates, so we have an "@name:" prefix instead. ```javascript [{ "id": "A", "@x:some_predicate": "B", "@y:some_predicate": "C" }] ``` Will only match if _both_ ```text A some_predicate B . A some_predicate C . ``` exist. This combines with the reversal rule to create paths like `"@a:!some_predicate"` ================================================ FILE: docs/quickstart-as-application.md ================================================ # Quickstart-As-Application See [Getting Started](https://github.com/cayleygraph/cayley/blob/master/docs/getting-started.md) and [Installation](https://github.com/cayleygraph/cayley/blob/master/docs/installation.md) ================================================ FILE: docs/quickstart-as-lib.md ================================================ # Quickstart as Library Currently, Cayley supports being used as a Go library for other projects. To use it in such a way, here's a quick example: ```go package main import ( "fmt" "log" "github.com/cayleygraph/cayley" "github.com/cayleygraph/quad" ) func main() { // Create a brand new graph store, err := cayley.NewMemoryGraph() if err != nil { log.Fatalln(err) } store.AddQuad(quad.Make("phrase of the day", "is of course", "Hello World!", nil)) // Now we create the path, to get to our data p := cayley.StartPath(store, quad.String("phrase of the day")).Out(quad.String("is of course")) // Now we iterate over results. Arguments: // 1. Optional context used for cancellation. // 2. Flag to optimize query before execution. // 3. Quad store, but we can omit it because we have already built path with it. err = p.Iterate(nil).EachValue(nil, func(value quad.Value){ nativeValue := quad.NativeOf(value) // this converts RDF values to normal Go types fmt.Println(nativeValue) }) if err != nil { log.Fatalln(err) } } ``` To use other backends, you can empty-import them, eg ```go import _ "github.com/cayleygraph/cayley/graph/kv/bolt" ``` And use them with a call like ```go import "github.com/cayleygraph/cayley/graph" func open() { // Initialize the database graph.InitQuadStore("bolt", path, nil) // Open and use the database cayley.NewGraph("bolt", path, nil) } ``` More runnable examples are available in [examples](https://github.com/cayleygraph/cayley/tree/87c9c341848b59924a054ebc2dd0f2bf8c57c6a9/examples/README.md) folder. ================================================ FILE: docs/todo.md ================================================ # TODOs The main source of our TODO list is our [Github Issues](https://github.com/cayleygraph/cayley/issues), so we are going to try to avoid duplicating them here. ## Anything marked "TODO" in the code. Usually something that should be taken care of. ================================================ FILE: docs/tools/convert-linked-data-files.md ================================================ --- description: >- Linked Data has multiple representations. The Cayley CLI includes a utility to convert Linked Data files from one format to another. --- # Convert Linked Data files ## Convert from one format to another ``` $ cayley convert -i data.jsonld -o data.nquads ``` `-i` is the input file to be converted. In this example it is a [JSON-LD](https://www.w3.org/TR/json-ld11/) file named `data.jsonld`. `-o` is the file to be created in the desired format. In this example it is a [N-Quads](https://www.w3.org/TR/n-quads/) file named `data.nquads`. ### Explicitly specify formats The formats of the input and output files are detected automatically by the file extension. In case a specific format should be used for input or output use `--load_format` and `--dump_format` respectively. ```text $ cayley convet -i data.jsonld -o data --dump_format pquads ``` `--dump_format` is set to the P-Quads format, a binary format used internally in Cayley. ================================================ FILE: docs/ui-overview.md ================================================ # UI Overview ## Sidebar Along the side are the various actions or views you can take. From the top, these are: * Run Query \(run the query\) * Gizmo \(a dropdown, to pick your query language, MQL is the other\) * [GizmoAPI.md](gizmoapi.md): This is the one of the two query languages used either via the REPL or HTTP interface. * [MQL.md](mql.md): The _other_ query language the interfaces support. * Query \(a request/response editor for the query language\) * Query Shape \(a visualization of the shape of the final query. Does not execute the query.\) * Visualize \(runs a query and, if tagged correctly, gives a sigmajs view of the results\) * Write \(an interface to write or remove individual quads or quad files\) * Documentation \(this documentation\) ## Visualize To use the visualize function, emit, either through tags or JS post-processing, a set of JSON objects containing the keys `source` and `target`. These will be the links, and nodes will automatically be detected. For example: ```javascript [ { source: "node1", target: "node2" }, { source: "node1", target: "node3" } ]; ``` Other keys are ignored. The upshot is that if you use the "Tag" functionality to add "source" and "target" tags, you can extract and quickly view subgraphs. ```text // Visualize who dani follows. g.V("").Tag("source").Out("").Tag("target").All() ``` The visualizer expects to tag nodes as either "source" or "target." Your source is represented as a blue node. While your target is represented as an orange node. The idea being that our node relationship goes from blue to orange \(source to target\). ================================================ FILE: docs/usage/3rd-party-apis.md ================================================ # 3rd-Party-APIs Various 3rd party APIs * **Clojure**: [https://github.com/wgb/cayley-clj](https://github.com/wgb/cayley-clj) * **Javascript/NodeJS**: [https://github.com/lnshi/node-cayley](https://github.com/lnshi/node-cayley), [https://github.com/villadora/cayley.js](https://github.com/villadora/cayley.js) * **Ruby**: [https://github.com/reneklacan/cayley-ruby](https://github.com/reneklacan/cayley-ruby) * **PHP**: [https://github.com/mcuadros/php-cayley](https://github.com/mcuadros/php-cayley) * **Python**: [https://github.com/ziyasal/pyley](https://github.com/ziyasal/pyley) * **.NET**: [https://github.com/ziyasal/Cayley.Net](https://github.com/ziyasal/Cayley.Net) * **Rust** \(in early stage\): [https://github.com/shamansir/cayley-rust](https://github.com/shamansir/cayley-rust) * **Haskell**: [https://github.com/MichelBoucey/cayley-client](https://github.com/MichelBoucey/cayley-client) ================================================ FILE: docs/usage/advanced-use.md ================================================ # Advanced Use ## Initialize A Graph Now that Cayley is downloaded \(or built\), let's create our database. `init` is the subcommand to set up a database and the right indices. You can set up a full [configuration file](../configuration.md) if you'd prefer, but it will also work from the command line. Examples for each backend can be found in `store.address` format from [config file](../configuration.md). Those two options \(db and dbpath\) are always going to be present. If you feel like not repeating yourself, setting up a configuration file for your backend might be something to do now. There's an example file, `cayley_example.yml` in the root directory. You can repeat the `--db (-i)` and `--dbpath (-a)` flags from here forward instead of the config flag, but let's assume you created `cayley_overview.yml` Note: when you specify parameters in the config file the config flags \(command line arguments\) are ignored. ## Load Data Into A Graph After the database is initialized we load the data. ```bash ./cayley load -c cayley_overview.yml -i data/testdata.nq ``` And wait. It will load. If you'd like to watch it load, you can run ```bash ./cayley load -c cayley_overview.yml -i data/testdata.nq --alsologtostderr=true ``` And watch the log output go by. If you plan to import a large dataset into Cayley and try multiple backends, it makes sense to first convert the dataset to Cayley-specific binary format by running: ```bash ./cayley conv -i dataset.nq.gz -o dataset.pq.gz ``` This will minimize parsing overhead on future imports and will compress dataset a bit better. ## Connect a REPL To Your Graph Now it's loaded. We can use Cayley now to connect to the graph. As you might have guessed, that command is: ```bash ./cayley repl -c cayley_overview.yml ``` Where you'll be given a `cayley>` prompt. It's expecting Gizmo/JS, but that can also be configured with a flag. New nodes and links can be added with the following command: ```bash cayley> :a subject predicate object label . ``` Removing links works similarly: ```bash cayley> :d subject predicate object . ``` This is great for testing, and ultimately also for scripting, but the real workhorse is the next step. Go ahead and give it a try: ```text // Simple math cayley> 2 + 2 // JavaScript syntax cayley> x = 2 * 8 cayley> x // See all the entities in this small follow graph. cayley> graph.Vertex().All() // See only dani. cayley> graph.Vertex("").All() // See who dani follows. cayley> graph.Vertex("").Out("").All() ``` ## Serve Your Graph Just as before: ```bash ./cayley http -c cayley_overview.yml ``` And you'll see a message not unlike ```bash listening on :64210, web interface at http://localhost:64210 ``` If you visit that address \(often, [http://localhost:64210](http://localhost:64210)\) you'll see the full web interface and also have a graph ready to serve queries via the [HTTP API](http.md) ### Access from other machines When you want to reach the API or UI from another machine in the network you need to specify the host argument: ```bash ./cayley http --config=cayley.cfg.overview --host=0.0.0.0:64210 ``` This makes it listen on all interfaces. You can also give it the specific the IP address you want Cayley to bind to. **Warning**: for security reasons you might not want to do this on a public accessible machine. ================================================ FILE: docs/usage/http.md ================================================ # HTTP Methods This file covers deprecated v1 HTTP API. All the methods of v2 HTTP API is described in OpenAPI/Swagger [spec](https://github.com/cayleygraph/cayley/tree/87c9c341848b59924a054ebc2dd0f2bf8c57c6a9/docs/api/swagger.yml) and can be viewed by importing `https://raw.githubusercontent.com/cayleygraph/cayley/master/docs/api/swagger.yml` URL into [Swagger Editor](https://editor.swagger.io/) or [Swagger UI demo](http://petstore.swagger.io/). ## Gephi Cayley supports streaming to Gephi via [GraphStream](../query-languages/gephigraphstream.md). ## API v1 Unless otherwise noted, all URIs take a POST command. ### Queries and Results #### `/api/v1/query/gizmo` POST Body: Javascript source code of the query Response: JSON results, depending on the query. #### `/api/v1/query/graphql` POST Body: [GraphQL](../query-languages/graphql.md) query Response: JSON results, depending on the query. #### `/api/v1/query/mql` POST Body: JSON MQL query Response: JSON results, with a query wrapper: ```javascript { "result": } ``` If the JSON is invalid or an error occurs: ```javascript { "error": "Error message" } ``` ### Query Shapes Result form: ```javascript { "nodes": [{ "id" : integer, "tags": ["list of tags from the query"], "values": ["known values from the query"], "is_link_node": bool, // Does the node represent the link or the node (the oval shapes) "is_fixed": bool // Is the node a fixed starting point of the query }], "links": [{ "source": integer, // Node ID "target": integer, // Node ID "link_node": integer // Node ID }] } ``` #### `/api/v1/shape/gizmo` POST Body: Javascript source code of the query Response: JSON description of the last query executed. #### `/api/v1/shape/mql` POST Body: JSON MQL query Response: JSON description of the query. ### Write commands Responses come in the form 200 Success: ```javascript { "result": "Success message." } ``` 400 / 500 Error: ```javascript { "error": "Error message." } ``` #### `/api/v1/write` POST Body: JSON quads ```javascript [{ "subject": "Subject Node", "predicate": "Predicate Node", "object": "Object node", "label": "Label node" // Optional }] // More than one quad allowed. ``` Response: JSON response message #### `/api/v1/write/file/nquad` POST Body: Form-encoded body: * Key: `NQuadFile`, Value: N-Quad file to write. Response: JSON response message Example: ```text curl http://localhost:64210/api/v1/write/file/nquad -F NQuadFile=@30k.n3 ``` #### `/api/v1/delete` POST Body: JSON quads ```javascript [{ "subject": "Subject Node", "predicate": "Predicate Node", "object": "Object node", "label": "Label node" // Optional }] // More than one quad allowed. ``` Response: JSON response message. ================================================ FILE: docs/usage/migration.md ================================================ # Migration ## From Cayley 0.6.1 to 0.7.x First you need to dump all the data from the database via v0.6.1: ```bash ./cayley-0.6 dump --db --dbpath
--dump_type pquads --dump ./data.pq.gz ``` or using config file: ```bash ./cayley-0.6 dump --config --dump_type pquads --dump ./data.pq.gz ``` And load the data into new database via v0.7.x \(`pq` file extension is important\): ```bash ./cayley load --init -d -a -i ./data.pq.gz ``` or using config file: ```bash ./cayley load --init -c -i ./data.pq.gz ``` ### Dump via text format An above guide uses Cayley-specific binary format to avoid encoding and parsing overhead and to compress output file better. As an alternative, a standard nquads file format can be used to dump and load data \(note `nq` extension\): ```bash ./cayley-0.6 dump --config --dump ./data.nq.gz ./cayley load --init -c -i ./data.nq.gz ``` ### Bolt, LevelDB, MongoDB Cayley v0.7.0 is still able to read and write databases of these types in old format. It can be accessed by changing backend name from `bolt`/`leveldb`/`mongo` to `bolt1`/`leveldb1`/`mongo1`. Thus, you can use guide for moving data between different backends for v0.7.x \(see below\). **Note:** support for old versions will be dropped starting from v0.7.1. ### SQL Data format for SQL between v0.6.1 and v0.7.x has not changed significantly. Database can be upgraded by executing the following SQL statements: ```sql ALTER TABLE quads DROP id; ALTER TABLE nodes ADD refs INT DEFAULT 0 NOT NULL; ALTER TABLE nodes ALTER COLUMN refs DROP DEFAULT; UPDATE nodes SET refs = (SELECT count(1) FROM quads WHERE subject_hash = nodes.hash) + (SELECT count(1) FROM quads WHERE predicate_hash = nodes.hash) + (SELECT count(1) FROM quads WHERE object_hash = nodes.hash) + (SELECT count(1) FROM quads WHERE label_hash = nodes.hash); ``` ## From different backend \(Cayley 0.7+\) First you need to dump all the data from old backend \(`pq` extension is important\): ```bash ./cayley dump -d -a
-o ./data.pq.gz ``` or using config file: ```bash ./cayley dump -c -o ./data.pq.gz ``` And load the data into a new backend and/or database: ```bash ./cayley load --init -d -a -i ./data.pq.gz ``` or using config file: ```bash ./cayley load --init -c -i ./data.pq.gz ``` ### Dump via text format An above guide uses Cayley-specific binary format to avoid encoding and parsing overhead and to compress output file better. As an alternative, a standard nquads file format can be used to dump and load data \(note `nq` extension\): ```bash ./cayley dump -c -o ./data.nq.gz ./cayley load --init -c -i ./data.nq.gz ``` ================================================ FILE: docs/usage/quickstart-as-lib.md ================================================ # Quickstart as Library Currently, Cayley supports being used as a Go library for other projects. To use it in such a way, here's a quick example: ```go package main import ( "fmt" "log" "github.com/cayleygraph/cayley" "github.com/cayleygraph/quad" ) func main() { // Create a brand new graph store, err := cayley.NewMemoryGraph() if err != nil { log.Fatalln(err) } store.AddQuad(quad.Make("phrase of the day", "is of course", "Hello World!", nil)) // Now we create the path, to get to our data p := cayley.StartPath(store, quad.String("phrase of the day")).Out(quad.String("is of course")) // Now we iterate over results. Arguments: // 1. Optional context used for cancellation. // 2. Flag to optimize query before execution. // 3. Quad store, but we can omit it because we have already built path with it. err = p.Iterate(nil).EachValue(nil, func(value quad.Value){ nativeValue := quad.NativeOf(value) // this converts RDF values to normal Go types fmt.Println(nativeValue) }) if err != nil { log.Fatalln(err) } } ``` To use other backends, you can empty-import them, eg ```go import _ "github.com/cayleygraph/cayley/graph/kv/bolt" ``` And use them with a call like ```go import "github.com/cayleygraph/cayley/graph" func open() { // Initialize the database graph.InitQuadStore("bolt", path, nil) // Open and use the database cayley.NewGraph("bolt", path, nil) } ``` More runnable examples are available in [examples](https://github.com/cayleygraph/cayley/tree/87c9c341848b59924a054ebc2dd0f2bf8c57c6a9/examples/README.md) folder. ================================================ FILE: docs/usage/ui-overview.md ================================================ # UI Overview ## Sidebar Along the side are the various actions or views you can take. From the top, these are: * Run Query \(run the query\) * Gizmo \(a dropdown, to pick your query language, MQL is the other\) * [GizmoAPI.md](../query-languages/gizmoapi.md): This is the one of the two query languages used either via the REPL or HTTP interface. * [MQL.md](../query-languages/mql.md): The _other_ query language the interfaces support. * Query \(a request/response editor for the query language\) * Query Shape \(a visualization of the shape of the final query. Does not execute the query.\) * Visualize \(runs a query and, if tagged correctly, gives a sigmajs view of the results\) * Write \(an interface to write or remove individual quads or quad files\) * Documentation \(this documentation\) ## Visualize To use the visualize function, emit, either through tags or JS post-processing, a set of JSON objects containing the keys `source` and `target`. These will be the links, and nodes will automatically be detected. For example: ```javascript [ { source: "node1", target: "node2" }, { source: "node1", target: "node3" } ]; ``` Other keys are ignored. The upshot is that if you use the "Tag" functionality to add "source" and "target" tags, you can extract and quickly view subgraphs. ```text // Visualize who dani follows. g.V("").Tag("source").Out("").Tag("target").All() ``` The visualizer expects to tag nodes as either "source" or "target." Your source is represented as a blue node. While your target is represented as an orange node. The idea being that our node relationship goes from blue to orange \(source to target\). ================================================ FILE: examples/README.md ================================================ # Examples Each of the examples can be run with ```go run hello_world/main.go``` obviously changing **hello_world** to the one you want to run! The hello_bolt example requires `go get`. ================================================ FILE: examples/hello_bolt/main.go ================================================ package main import ( "context" "fmt" "io/ioutil" "log" "os" "github.com/cayleygraph/cayley" "github.com/cayleygraph/cayley/graph" _ "github.com/cayleygraph/cayley/graph/kv/bolt" "github.com/cayleygraph/quad" ) func main() { // File for your new BoltDB. Use path to regular file and not temporary in the real world tmpdir, err := ioutil.TempDir("", "example") if err != nil { log.Fatal(err) } defer os.RemoveAll(tmpdir) // clean up // Initialize the database err = graph.InitQuadStore("bolt", tmpdir, nil) if err != nil { log.Fatal(err) } // Open and use the database store, err := cayley.NewGraph("bolt", tmpdir, nil) if err != nil { log.Fatalln(err) } store.AddQuad(quad.Make("phrase of the day", "is of course", "Hello BoltDB!", "demo graph")) // Now we create the path, to get to our data p := cayley.StartPath(store, quad.String("phrase of the day")).Out(quad.String("is of course")) // This is more advanced example of the query. // Simpler equivalent can be found in hello_world example. ctx := context.TODO() // Now we get an iterator for the path and optimize it. // The second return is if it was optimized, but we don't care for now. its, _ := p.BuildIterator(ctx).Optimize(ctx) it := its.Iterate() // remember to cleanup after yourself defer it.Close() // While we have items for it.Next(ctx) { token := it.Result() // get a ref to a node (backend-specific) value, err := store.NameOf(token) // get the value in the node (RDF) if err != nil { log.Fatalln(err) } nativeValue := quad.NativeOf(value) // convert value to normal Go type fmt.Println(nativeValue) // print it! } if err := it.Err(); err != nil { log.Fatalln(err) } } ================================================ FILE: examples/hello_schema/main.go ================================================ package main import ( "context" "fmt" "io/ioutil" "log" "math/rand" "os" "github.com/cayleygraph/cayley" "github.com/cayleygraph/cayley/graph" _ "github.com/cayleygraph/cayley/graph/kv/bolt" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" // Import RDF vocabulary definitions to be able to expand IRIs like rdf:label. _ "github.com/cayleygraph/quad/voc/core" ) type Person struct { // dummy field to enforce all object to have a relation // means nothing for Go itself rdfType struct{} `quad:"@type > ex:Person"` ID quad.IRI `json:"@id"` // tag @id is a special one - graph node value will be stored in this field Name string `json:"ex:name"` // field name (predicate) may be written as json field name Age int `quad:"ex:age"` // or in a quad tag } type Coords struct { // Object may be without id - it will be generated automatically. // It's also not necessary to have a type definition. Lat float64 `json:"ex:lat"` Lng float64 `json:"ex:lng"` } func checkErr(err error) { if err != nil { log.Fatal(err) } } func main() { // Define an "ex:" prefix for IRIs that will be expanded to "http://example.org". // "ex:name" will become "http://example.org/name" voc.RegisterPrefix("ex:", "http://example.org/") // Associate Go type with an IRI. // All Coords objects will now generate a triple. schema.RegisterType(quad.IRI("ex:Coords"), Coords{}) sch := schema.NewConfig() // Override a function to generate IDs. Can be changed to generate UUIDs, for example. sch.GenerateID = func(_ interface{}) quad.Value { return quad.BNode(fmt.Sprintf("node%d", rand.Intn(1000))) } // File for your new BoltDB. Use path to regular file and not temporary in the real world tmpdir, err := ioutil.TempDir("", "example") checkErr(err) defer os.RemoveAll(tmpdir) // clean up // Initialize the database err = graph.InitQuadStore("bolt", tmpdir, nil) checkErr(err) // Open and use the database store, err := cayley.NewGraph("bolt", tmpdir, nil) checkErr(err) defer store.Close() qw := graph.NewWriter(store) // Save an object bob := Person{ ID: quad.IRI("ex:bob").Full().Short(), Name: "Bob", Age: 32, } fmt.Printf("saving: %+v\n", bob) id, err := sch.WriteAsQuads(qw, bob) checkErr(err) err = qw.Close() checkErr(err) fmt.Println("id for object:", id, "=", bob.ID) // should be equal // Get object by id var someone Person err = sch.LoadTo(nil, store, &someone, id) checkErr(err) fmt.Printf("loaded: %+v\n", someone) // Or get all objects of type Person var people []Person err = sch.LoadTo(nil, store, &people) checkErr(err) fmt.Printf("people: %+v\n", people) fmt.Println() // Store objects with no ID and type coords := []Coords{ {Lat: 12.3, Lng: 34.5}, {Lat: 39.7, Lng: 8.41}, } qw = graph.NewWriter(store) for _, c := range coords { id, err = sch.WriteAsQuads(qw, c) checkErr(err) fmt.Println("generated id:", id) } err = qw.Close() checkErr(err) // Get coords back var newCoords []Coords err = sch.LoadTo(nil, store, &newCoords) checkErr(err) fmt.Printf("coords: %+v\n", newCoords) // Print quads fmt.Println("\nquads:") ctx := context.TODO() it := store.QuadsAllIterator().Iterate() defer it.Close() for it.Next(ctx) { fmt.Println(store.Quad(it.Result())) } } ================================================ FILE: examples/hello_world/main.go ================================================ package main import ( "fmt" "log" "github.com/cayleygraph/cayley" "github.com/cayleygraph/quad" ) func main() { // Create a brand new graph store, err := cayley.NewMemoryGraph() if err != nil { log.Fatalln(err) } store.AddQuad(quad.Make("phrase of the day", "is of course", "Hello World!", nil)) // Now we create the path, to get to our data p := cayley.StartPath(store, quad.String("phrase of the day")).Out(quad.String("is of course")) // Now we iterate over results. Arguments: // 1. Optional context used for cancellation. // 2. Quad store, but we can omit it because we have already built path with it. err = p.Iterate(nil).EachValue(nil, func(value quad.Value) error { nativeValue := quad.NativeOf(value) // this converts RDF values to normal Go types fmt.Println(nativeValue) return nil }) if err != nil { log.Fatalln(err) } } ================================================ FILE: examples/transaction/main.go ================================================ package main import ( "fmt" "log" "github.com/cayleygraph/cayley" "github.com/cayleygraph/quad" ) func main() { // To see how most of this works, see hello_world -- this just add in a transaction store, err := cayley.NewMemoryGraph() if err != nil { log.Fatalln(err) } // Create a transaction of work to do // NOTE: the transaction is independent of the storage type, so comes from cayley rather than store t := cayley.NewTransaction() t.AddQuad(quad.Make("food", "is", "good", nil)) t.AddQuad(quad.Make("phrase of the day", "is of course", "Hello World!", nil)) t.AddQuad(quad.Make("cats", "are", "awesome", nil)) t.AddQuad(quad.Make("cats", "are", "scary", nil)) t.AddQuad(quad.Make("cats", "want to", "kill you", nil)) // Apply the transaction err = store.ApplyTransaction(t) if err != nil { log.Fatalln(err) } p := cayley.StartPath(store, quad.String("cats")).Out(quad.String("are")) err = p.Iterate(nil).EachValue(nil, func(v quad.Value) error { fmt.Println("cats are", v.Native()) return nil }) if err != nil { log.Fatalln(err) } } ================================================ FILE: go.mod ================================================ module github.com/cayleygraph/cayley go 1.22 toolchain go1.22.5 require ( github.com/badgerodon/peg v0.0.0-20130729175151-9e5f7f4d07ca github.com/cayleygraph/quad v1.3.0 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/dennwc/graphql v0.0.0-20180603144102-12cfed44bc5d github.com/dop251/goja v0.0.0-20240627195025-eb1f15ee67d2 github.com/fsouza/go-dockerclient v1.11.0 github.com/go-sql-driver/mysql v1.8.1 github.com/golang/glog v1.2.1 github.com/hidal-go/hidalgo v0.3.0 github.com/jackc/pgx/v5 v5.6.0 github.com/julienschmidt/httprouter v1.3.0 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.22 github.com/peterh/liner v1.2.2 github.com/piprate/json-gold v0.5.0 github.com/prometheus/client_golang v1.19.1 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/syndtr/goleveldb v1.0.0 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 golang.org/x/net v0.27.0 google.golang.org/appengine v1.6.8 google.golang.org/protobuf v1.34.2 ) require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/DataDog/zstd v1.5.0 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.9.0 // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/pebble v0.0.0-20220318150003-0ad186894f6d // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/containerd/containerd v1.7.19 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/log v0.1.0 // indirect github.com/d4l3k/messagediff v1.2.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/docker/docker v27.0.3+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getsentry/sentry-go v0.13.0 // indirect github.com/go-kivik/couchdb v2.0.0+incompatible // indirect github.com/go-kivik/kivik v2.0.0+incompatible // indirect github.com/go-kivik/pouchdb v2.0.1+incompatible // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 // indirect github.com/gopherjs/jsbuiltin v0.0.0-20180426082241-50091555e127 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/user v0.1.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.1.13 // indirect github.com/ory/dockertest v3.3.5+incompatible // indirect github.com/otiai10/copy v1.12.0 // indirect github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/cachecontrol v0.2.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.1 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect go.etcd.io/bbolt v1.3.10 // indirect go.mongodb.org/mongo-driver v1.8.4 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/olivere/elastic.v5 v5.0.86 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.0 // indirect ) replace github.com/Sirupsen/logrus => github.com/Sirupsen/logrus v1.0.1 ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/badgerodon/peg v0.0.0-20130729175151-9e5f7f4d07ca h1:77KAMse6RWRpPfVnIZcAtJ/5ZK/oRCeY94ZjIWSbe0g= github.com/badgerodon/peg v0.0.0-20130729175151-9e5f7f4d07ca/go.mod h1:TWe0N2hv5qvpLHT+K16gYcGBllld4h65dQ/5CNuirmk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cayleygraph/quad v1.3.0 h1:xg7HOLWWPgvZ4CcvzEpfCwq42L8mzYUR+8V0jtYoBzc= github.com/cayleygraph/quad v1.3.0/go.mod h1:NadtM7uMm78FskmX++XiOOrNvgkq0E1KvvhQdMseMz4= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0= github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88= github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/pebble v0.0.0-20220318150003-0ad186894f6d h1:VHoyzrRXf6pO4+3qyR7T0ez9rYT5GLTDcKk0epdumt4= github.com/cockroachdb/pebble v0.0.0-20220318150003-0ad186894f6d/go.mod h1:buxOO9GBtOcq1DiXDpIPYrmxY020K2A8lOrwno5FetU= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/containerd/containerd v1.7.19 h1:/xQ4XRJ0tamDkdzrrBAUy/LE5nCcxFKdBm4EcPrSMEE= github.com/containerd/containerd v1.7.19/go.mod h1:h4FtNYUUMB4Phr6v+xG89RYKj9XccvbNSCKjdufCrkc= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dennwc/graphql v0.0.0-20180603144102-12cfed44bc5d h1:QWlaiMNg63HE5qimJd4stjg9l1Ca4BKcgs+UNSWPJ+s= github.com/dennwc/graphql v0.0.0-20180603144102-12cfed44bc5d/go.mod h1:lg9KQn0BgRCSCGNpcGvJp/0Ljf1Yxk8TZq9HSYc43fk= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE= github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dop251/goja v0.0.0-20240627195025-eb1f15ee67d2 h1:4Ew88p5s9dwIk5/woUyqI9BD89NgZoUNH4/rM/h2UDg= github.com/dop251/goja v0.0.0-20240627195025-eb1f15ee67d2/go.mod h1:o31y53rb/qiIAONF7w3FHJZRqqP3fzHUr1HqanthByw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flimzy/diff v0.1.7 h1:DRbd+lN3lY1xVuQrfqvDNsqBwA6RMbClMs6tS5sqWWk= github.com/flimzy/diff v0.1.7/go.mod h1:lFJtC7SPsK0EroDmGTSrdtWKAxOk3rO+q+e04LL05Hs= github.com/flimzy/testy v0.1.17 h1:Y+TUugY6s4B/vrOEPo6SUKafc41W5aiX3qUWvhAPMdI= github.com/flimzy/testy v0.1.17/go.mod h1:3szguN8NXqgq9bt9Gu8TQVj698PJWmyx/VY1frwwKrM= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fsouza/go-dockerclient v1.11.0 h1:4ZAk6W7rPAtPXm7198EFqA5S68rwnNQORxlOA5OurCA= github.com/fsouza/go-dockerclient v1.11.0/go.mod h1:0I3TQCRseuPTzqlY4Y3ajfsg2VAdMQoazrkxJTiJg8s= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kivik/couchdb v2.0.0+incompatible h1:DsXVuGJTng04Guz8tg7jGVQ53RlByEhk+gPB/1yo3Oo= github.com/go-kivik/couchdb v2.0.0+incompatible/go.mod h1:5XJRkAMpBlEVA4q0ktIZjUPYBjoBmRoiWvwUBzP3BOQ= github.com/go-kivik/kivik v2.0.0+incompatible h1:/7hgr29DKv/vlaJsUoyRlOFq0K+3ikz0wTbu+cIs7QY= github.com/go-kivik/kivik v2.0.0+incompatible/go.mod h1:nIuJ8z4ikBrVUSk3Ua8NoDqYKULPNjuddjqRvlSUyyQ= github.com/go-kivik/kiviktest v2.0.0+incompatible h1:y1RyPHqWQr+eFlevD30Tr3ipiPCxK78vRoD3o9YysjI= github.com/go-kivik/kiviktest v2.0.0+incompatible/go.mod h1:JdhVyzixoYhoIDUt6hRf1yAfYyaDa5/u9SDOindDkfQ= github.com/go-kivik/pouchdb v2.0.1+incompatible h1:v3OWB0/56qXjdlmu9az76nJ5utlbsHyRuB3uRq4sWFg= github.com/go-kivik/pouchdb v2.0.1+incompatible/go.mod h1:U+siUrqLCVxeMU3QjQTYIC3/F/e6EUKm+o5buJb7vpw= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 h1:QJq7UBOuoynsywLk+aC75rC2Cbi2+lQRDaLaizhA+fA= github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gopherjs/jsbuiltin v0.0.0-20180426082241-50091555e127 h1:atBEgNR1C5+LFkl8ipQtLee9RStheS8YeCSkiYqBhOg= github.com/gopherjs/jsbuiltin v0.0.0-20180426082241-50091555e127/go.mod h1:7X1acUyFRf+oVFTU6SWw9mnb57Vxn+Nbh8iPbKg95hs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hidal-go/hidalgo v0.3.0 h1:z74GAu6SGbznyEcLbsiFpsFNkcQcSZnKIXdn9Wz85Ek= github.com/hidal-go/hidalgo v0.3.0/go.mod h1:N85Y5d1N68Y1gJ3SYyQ/wHbMdHDZ/yH8Bp3hht1iuPY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olivere/elastic/v7 v7.0.12/go.mod h1:14rWX28Pnh3qCKYRVnSGXWLf9MbLonYS/4FDCY3LAPo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0 h1:M76yO2HkZASFjXL0HSoZJ1AYEmQxNJmY41Jx1zNUq1Y= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.13 h1:98S2srgG9vw0zWcDpFMn5TRrh8kLxa/5OFUstuUhmRs= github.com/opencontainers/runc v1.1.13/go.mod h1:R016aXacfp/gwQBYw2FDGa9m+n6atbLWrYY8hNMT/sA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/piprate/json-gold v0.5.0 h1:RmGh1PYboCFcchVFuh2pbSWAZy4XJaqTMU4KQYsApbM= github.com/piprate/json-gold v0.5.0/go.mod h1:WZ501QQMbZZ+3pXFPhQKzNwS1+jls0oqov3uQ2WasLs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k= github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= gitlab.com/flimzy/testy v0.10.2 h1:GQq5CtJK0+ph2c2cD/FAzvTz/OJbxjpMr6Kaf/UerHg= gitlab.com/flimzy/testy v0.10.2/go.mod h1:tcu652e6AyD5wS8q2JRUI+j5SlwIYsl3yq3ulHyuh8M= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.mongodb.org/mongo-driver v1.8.4 h1:NruvZPPL0PBcRJKmbswoWSrmHeUvzdxA3GCPfD/NEOA= go.mongodb.org/mongo-driver v1.8.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8cqfWhK+68= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= ================================================ FILE: gogen.go ================================================ package cayley //go:generate go run ./cmd/docgen/docgen.go -i ./docs/GizmoAPI.md.in -o ./docs/GizmoAPI.md ================================================ FILE: graph/all/all.go ================================================ package all import ( // supported backends _ "github.com/cayleygraph/cayley/graph/kv/all" _ "github.com/cayleygraph/cayley/graph/memstore" _ "github.com/cayleygraph/cayley/graph/nosql/all" _ "github.com/cayleygraph/cayley/graph/sql/cockroach" _ "github.com/cayleygraph/cayley/graph/sql/mysql" _ "github.com/cayleygraph/cayley/graph/sql/postgres" ) ================================================ FILE: graph/all/all_cgo.go ================================================ //go:build cgo package all import ( // backends requiring cgo _ "github.com/cayleygraph/cayley/graph/sql/sqlite" ) ================================================ FILE: graph/gaedatastore/config.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gaedatastore import ( "encoding/json" "fmt" "os" "strconv" "time" ) // Config defines the behavior of cayley database instances. type Config struct { DatabaseType string DatabasePath string DatabaseOptions map[string]interface{} ReplicationType string ReplicationOptions map[string]interface{} ListenHost string ListenPort string ReadOnly bool Timeout time.Duration LoadSize int } type config struct { DatabaseType string `json:"database"` DatabasePath string `json:"db_path"` DatabaseOptions map[string]interface{} `json:"db_options"` ReplicationType string `json:"replication"` ReplicationOptions map[string]interface{} `json:"replication_options"` ListenHost string `json:"listen_host"` ListenPort string `json:"listen_port"` ReadOnly bool `json:"read_only"` Timeout duration `json:"timeout"` LoadSize int `json:"load_size"` } func (c *Config) UnmarshalJSON(data []byte) error { var t config err := json.Unmarshal(data, &t) if err != nil { return err } *c = Config{ DatabaseType: t.DatabaseType, DatabasePath: t.DatabasePath, DatabaseOptions: t.DatabaseOptions, ReplicationType: t.ReplicationType, ReplicationOptions: t.ReplicationOptions, ListenHost: t.ListenHost, ListenPort: t.ListenPort, ReadOnly: t.ReadOnly, Timeout: time.Duration(t.Timeout), LoadSize: t.LoadSize, } return nil } func (c *Config) MarshalJSON() ([]byte, error) { return json.Marshal(config{ DatabaseType: c.DatabaseType, DatabasePath: c.DatabasePath, DatabaseOptions: c.DatabaseOptions, ReplicationType: c.ReplicationType, ReplicationOptions: c.ReplicationOptions, ListenHost: c.ListenHost, ListenPort: c.ListenPort, ReadOnly: c.ReadOnly, Timeout: duration(c.Timeout), LoadSize: c.LoadSize, }) } // duration is a time.Duration that satisfies the // json.UnMarshaler and json.Marshaler interfaces. type duration time.Duration // UnmarshalJSON unmarshals a duration according to the following scheme: // * If the element is absent the duration is zero. // * If the element is parsable as a time.Duration, the parsed value is kept. // * If the element is parsable as a number, that number of seconds is kept. func (d *duration) UnmarshalJSON(data []byte) error { if len(data) == 0 { *d = 0 return nil } text := string(data) t, err := time.ParseDuration(text) if err == nil { *d = duration(t) return nil } i, err := strconv.ParseInt(text, 10, 64) if err == nil { *d = duration(time.Duration(i) * time.Second) return nil } // This hack is to get around strconv.ParseFloat // not handling e-notation for integers. f, err := strconv.ParseFloat(text, 64) *d = duration(time.Duration(f) * time.Second) return err } func (d *duration) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf("%q", *d)), nil } // LoadConf reads a JSON-encoded config contained in the given file. A zero value // config is returned if the filename is empty. func LoadConf(file string) (*Config, error) { config := &Config{} if file == "" { return config, nil } f, err := os.Open(file) if err != nil { return nil, fmt.Errorf("could not open config file %q: %v", file, err) } defer f.Close() dec := json.NewDecoder(f) err = dec.Decode(config) if err != nil { return nil, fmt.Errorf("could not parse config file %q: %v", file, err) } return config, nil } ================================================ FILE: graph/gaedatastore/iterator.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gaedatastore import ( "context" "fmt" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" "google.golang.org/appengine/datastore" ) var _ iterator.Shape = &Iterator{} const ( bufferSize = 50 ) type Iterator struct { size int64 dir quad.Direction qs *QuadStore t *Token isAll bool kind string } func (it *Iterator) Iterate() iterator.Scanner { if it.isAll { return newAllIteratorNext(it.qs, it.kind) } return newIteratorNext(it.qs, it.kind, it.dir, it.t) } func (it *Iterator) Lookup() iterator.Index { if it.isAll { return newAllIteratorContains(it.qs, it.kind) } return newIteratorContains(it.qs, it.kind, it.dir, it.t) } func (qs *QuadStore) newIterator(k string, d quad.Direction, val graph.Ref) *Iterator { t := val.(*Token) if t == nil { clog.Errorf("Token == nil") } return &Iterator{ dir: d, qs: qs, isAll: false, t: t, kind: k, } } func (qs *QuadStore) newAllIterator(kind string) *Iterator { return &Iterator{ qs: qs, dir: quad.Any, isAll: true, kind: kind, } } // No subiterators. func (it *Iterator) SubIterators() []iterator.Shape { return nil } func (it *Iterator) Sorted() bool { return false } func (it *Iterator) Optimize(ctx context.Context) (iterator.Shape, bool) { return it, false } func (it *Iterator) String() string { name := "" if it.t != nil { tn, err := it.qs.NameOf(it.t) if err != nil { name = "ERROR(" + err.Error() + ")" } else { name = quad.StringOf(tn) + "/" + it.t.Hash } } return fmt.Sprintf("GAE(%s)", name) } func (it *Iterator) getSize(ctx context.Context) (refs.Size, error) { if it.size != 0 { return refs.Size{ Value: it.size, Exact: true, }, nil } if !it.isAll { // The number of references to this node is held in the nodes entity key := it.qs.createKeyFromToken(it.t) foundNode := new(NodeEntry) err := datastore.Get(it.qs.context, key, foundNode) if err != datastore.ErrNoSuchEntity { err = nil } size := foundNode.Size it.size = size return refs.Size{ Value: it.size, Exact: err == nil, }, err } var size int64 st, err := it.qs.Stats(context.Background(), true) if it.kind == nodeKind { size = st.Nodes.Value } else { size = st.Quads.Value } it.size = size return refs.Size{ Value: it.size, Exact: err == nil, }, err } func (it *Iterator) Stats(ctx context.Context) (iterator.Costs, error) { sz, err := it.getSize(ctx) // TODO (panamafrancis) calculate costs return iterator.Costs{ ContainsCost: 1, NextCost: 5, Size: sz, }, err } type iteratorNext struct { dir quad.Direction qs *QuadStore name string isAll bool kind string hash string done bool buffer []string offset int last string result graph.Ref err error } func newIteratorNext(qs *QuadStore, k string, d quad.Direction, t *Token) *iteratorNext { if t == nil { clog.Errorf("Token == nil") } if t.Kind != nodeKind { clog.Errorf("Cannot create an iterator from a non-node value") return &iteratorNext{done: true} } if k != nodeKind && k != quadKind { clog.Errorf("Cannot create iterator for unknown kind") return &iteratorNext{done: true} } if qs.context == nil { clog.Errorf("Cannot create iterator without a valid context") return &iteratorNext{done: true} } tn, err := qs.NameOf(t) if err != nil { clog.Errorf("Creating iterator token lookup error: %v", err) return &iteratorNext{done: true, err: err} } name := quad.StringOf(tn) return &iteratorNext{ name: name, dir: d, qs: qs, isAll: false, kind: k, hash: t.Hash, done: false, } } func newAllIteratorNext(qs *QuadStore, kind string) *iteratorNext { if kind != nodeKind && kind != quadKind { clog.Errorf("Cannot create iterator for an unknown kind") return &iteratorNext{done: true} } if qs.context == nil { clog.Errorf("Cannot create iterator without a valid context") return &iteratorNext{done: true} } return &iteratorNext{ qs: qs, dir: quad.Any, isAll: true, kind: kind, } } func (it *iteratorNext) Close() error { it.buffer = nil it.offset = 0 it.done = true it.last = "" it.result = nil return nil } func (it *iteratorNext) TagResults(dst map[string]graph.Ref) {} func (it *iteratorNext) NextPath(ctx context.Context) bool { return false } func (it *iteratorNext) Result() graph.Ref { return it.result } func (it *iteratorNext) Next(ctx context.Context) bool { if it.offset+1 < len(it.buffer) { it.offset++ it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]} return true } if it.done { return false } // Reset buffer and offset it.offset = 0 it.buffer = make([]string, 0, bufferSize) // Create query // TODO (panamafrancis) Keys only query? q := datastore.NewQuery(it.kind).Limit(bufferSize) if !it.isAll { // Filter on the direction {subject,objekt...} q = q.Filter(it.dir.String()+" =", it.name) } // Get last cursor position cursor, err := datastore.DecodeCursor(it.last) if err == nil { q = q.Start(cursor) } // Buffer the keys of the next 50 matches t := q.Run(it.qs.context) for { // Quirk of the datastore, you cannot pass a nil value to to Next() // even if you just want the keys var k *datastore.Key skip := false if it.kind == quadKind { temp := new(QuadEntry) k, err = t.Next(temp) // Skip if quad has been deleted if len(temp.Added) <= len(temp.Deleted) { skip = true } } else { temp := new(NodeEntry) k, err = t.Next(temp) // Skip if node has been deleted if temp.Size == 0 { skip = true } } if err == datastore.Done { it.done = true break } if err != nil { clog.Errorf("Error fetching next entry %v", err) it.err = err return false } if !skip { it.buffer = append(it.buffer, k.StringID()) } } // Save cursor position cursor, err = t.Cursor() if err == nil { it.last = cursor.String() } // Protect against bad queries if it.done && len(it.buffer) == 0 { clog.Warningf("Query did not return any results") return false } // First result it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]} return true } func (it *iteratorNext) Err() error { return it.err } func (it *iteratorNext) Sorted() bool { return false } func (it *iteratorNext) String() string { return fmt.Sprintf("GAE(%s/%s)", it.name, it.hash) } type iteratorContains struct { dir quad.Direction qs *QuadStore name string isAll bool kind string hash string done bool result graph.Ref err error } func newIteratorContains(qs *QuadStore, k string, d quad.Direction, t *Token) *iteratorContains { if t == nil { clog.Errorf("Token == nil") } if t.Kind != nodeKind { clog.Errorf("Cannot create an iterator from a non-node value") return &iteratorContains{done: true} } if k != nodeKind && k != quadKind { clog.Errorf("Cannot create iterator for unknown kind") return &iteratorContains{done: true} } if qs.context == nil { clog.Errorf("Cannot create iterator without a valid context") return &iteratorContains{done: true} } tn, err := qs.NameOf(t) if err != nil { clog.Errorf("Creating iterator token lookup error: %v", err) return &iteratorContains{done: true, err: err} } name := quad.StringOf(tn) return &iteratorContains{ name: name, dir: d, qs: qs, isAll: false, kind: k, hash: t.Hash, done: false, } } func newAllIteratorContains(qs *QuadStore, kind string) *iteratorContains { if kind != nodeKind && kind != quadKind { clog.Errorf("Cannot create iterator for an unknown kind") return &iteratorContains{done: true} } if qs.context == nil { clog.Errorf("Cannot create iterator without a valid context") return &iteratorContains{done: true} } return &iteratorContains{ qs: qs, dir: quad.Any, isAll: true, kind: kind, } } func (it *iteratorContains) Close() error { it.done = true it.result = nil return nil } func (it *iteratorContains) Contains(ctx context.Context, v graph.Ref) bool { if it.isAll { // The result needs to be set, so when contains is called, the result can be retrieved it.result = v return true } t := v.(*Token) if t == nil { clog.Errorf("Could not cast to token") return false } if t.Kind == nodeKind { clog.Errorf("Contains does not work with node values") return false } // Contains is for when you want to know that an iterator refers to a quad var offset int switch it.dir { case quad.Subject: offset = 0 case quad.Predicate: offset = (quad.HashSize * 2) case quad.Object: offset = (quad.HashSize * 2) * 2 case quad.Label: offset = (quad.HashSize * 2) * 3 } val := t.Hash[offset : offset+(quad.HashSize*2)] if val == it.hash { return true } return false } func (it *iteratorContains) TagResults(dst map[string]graph.Ref) {} func (it *iteratorContains) NextPath(ctx context.Context) bool { return false } func (it *iteratorContains) Result() graph.Ref { return it.result } func (it *iteratorContains) Err() error { return it.err } func (it *iteratorContains) Sorted() bool { return false } func (it *iteratorContains) String() string { return fmt.Sprintf("GAE(%s/%s)", it.name, it.hash) } ================================================ FILE: graph/gaedatastore/quadstore.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gaedatastore import ( "encoding/hex" "errors" "math" "net/http" "time" "golang.org/x/net/context" "google.golang.org/appengine" "google.golang.org/appengine/datastore" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" httpgraph "github.com/cayleygraph/cayley/graph/http" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) var _ httpgraph.QuadStore = (*QuadStore)(nil) const ( QuadStoreType = "gaedatastore" quadKind = "quad" nodeKind = "node" ) var ( // Order of quad fields spo = [4]quad.Direction{quad.Subject, quad.Predicate, quad.Object, quad.Label} ) type QuadStore struct { context context.Context } type MetadataEntry struct { NodeCount int64 QuadCount int64 } type Token struct { Kind string Hash string } func (t Token) IsNode() bool { return t.Kind == nodeKind } func (t Token) Key() interface{} { return t } type QuadEntry struct { Hash string Added []int64 `datastore:",noindex"` Deleted []int64 `datastore:",noindex"` Subject string `datastore:"subject"` Predicate string `datastore:"predicate"` Object string `datastore:"object"` Label string `datastore:"label"` } type NodeEntry struct { Name string Size int64 } type LogEntry struct { Action string Key string Timestamp int64 } func init() { graph.RegisterQuadStore("gaedatastore", graph.QuadStoreRegistration{ NewFunc: newQuadStore, UpgradeFunc: nil, InitFunc: initQuadStore, IsPersistent: true, }) } func initQuadStore(_ string, _ graph.Options) error { // TODO (panamafrancis) check appengine datastore for consistency return nil } func newQuadStore(_ string, options graph.Options) (graph.QuadStore, error) { return &QuadStore{}, nil } func (qs *QuadStore) createKeyForQuad(q quad.Quad) *datastore.Key { id := hashOf(q.Subject) id += hashOf(q.Predicate) id += hashOf(q.Object) id += hashOf(q.Label) return qs.createKeyFromToken(&Token{quadKind, id}) } func hashOf(s quad.Value) string { return hex.EncodeToString(quad.HashOf(s)) } func (qs *QuadStore) createKeyForNode(n quad.Value) *datastore.Key { id := hashOf(n) return qs.createKeyFromToken(&Token{nodeKind, id}) } func (qs *QuadStore) createKeyForMetadata() *datastore.Key { return qs.createKeyFromToken(&Token{"metadata", "metadataentry"}) } func (qs *QuadStore) createKeyForLog() *datastore.Key { return datastore.NewKey(qs.context, "logentry", "", 0, nil) } func (qs *QuadStore) createKeyFromToken(t *Token) *datastore.Key { return datastore.NewKey(qs.context, t.Kind, t.Hash, 0, nil) } func (qs *QuadStore) checkValid(k *datastore.Key) (bool, error) { var q QuadEntry err := datastore.Get(qs.context, k, &q) if err == datastore.ErrNoSuchEntity { return false, nil } if _, ok := err.(*datastore.ErrFieldMismatch); ok { return true, nil } if err != nil { clog.Warningf("Error occurred when getting quad/node %s %v", k, err) return false, err } // a deleted node should not be returned as found. if len(q.Deleted) >= len(q.Added) { return false, nil } return true, nil } func getContext(opts graph.Options) (context.Context, error) { req := opts["HTTPRequest"].(*http.Request) if req == nil { err := errors.New("HTTP Request needed") clog.Errorf("%v", err) return nil, err } return appengine.NewContext(req), nil } func (qs *QuadStore) ForRequest(r *http.Request) (graph.QuadStore, error) { return &QuadStore{context: appengine.NewContext(r)}, nil } func (qs *QuadStore) NewQuadWriter() (quad.WriteCloser, error) { return &quadWriter{qs: qs}, nil } type quadWriter struct { qs *QuadStore deltas []graph.Delta } func (w *quadWriter) WriteQuad(q quad.Quad) error { _, err := w.WriteQuads([]quad.Quad{q}) return err } func (w *quadWriter) WriteQuads(buf []quad.Quad) (int, error) { // TODO(dennwc): write an optimized implementation w.deltas = w.deltas[:0] if cap(w.deltas) < len(buf) { w.deltas = make([]graph.Delta, 0, len(buf)) } for _, q := range buf { w.deltas = append(w.deltas, graph.Delta{ Quad: q, Action: graph.Add, }) } err := w.qs.ApplyDeltas(w.deltas, graph.IgnoreOpts{ IgnoreDup: true, }) w.deltas = w.deltas[:0] if err != nil { return 0, err } return len(buf), nil } func (w *quadWriter) Close() error { w.deltas = nil return nil } func (qs *QuadStore) ApplyDeltas(in []graph.Delta, ignoreOpts graph.IgnoreOpts) error { if qs.context == nil { return errors.New("No context, graph not correctly initialised") } toKeep := make([]graph.Delta, 0) for _, d := range in { if d.Action != graph.Add && d.Action != graph.Delete { //Defensive shortcut return errors.New("Datastore: invalid action") } key := qs.createKeyForQuad(d.Quad) keep := false switch d.Action { case graph.Add: found, err := qs.checkValid(key) if err != nil { return err } if found { if !ignoreOpts.IgnoreDup { return graph.ErrQuadExists } } else { keep = true } case graph.Delete: found, err := qs.checkValid(key) if err != nil { return err } if found || ignoreOpts.IgnoreMissing { keep = true } else { return graph.ErrQuadNotExist } default: keep = false } if keep { toKeep = append(toKeep, d) } } if len(toKeep) == 0 { return nil } ids, err := qs.updateLog(toKeep) if err != nil { clog.Errorf("Updating log failed %v", err) return err } if clog.V(2) { clog.Infof("Existence verified. Proceeding.") } quadsAdded, err := qs.updateQuads(toKeep, ids) if err != nil { clog.Errorf("UpdateQuads failed %v", err) return err } nodesAdded, err := qs.updateNodes(toKeep) if err != nil { clog.Warningf("UpdateNodes failed %v", err) return err } err = qs.updateMetadata(quadsAdded, nodesAdded) if err != nil { clog.Warningf("UpdateMetadata failed %v", err) return err } return nil } func (qs *QuadStore) updateNodes(in []graph.Delta) (int64, error) { // Collate changes to each node var countDelta int64 var nodesAdded int64 nodeDeltas := make(map[quad.Value]int64) for _, d := range in { if d.Action == graph.Add { countDelta = 1 } else { countDelta = -1 } nodeDeltas[d.Quad.Subject] += countDelta nodeDeltas[d.Quad.Object] += countDelta nodeDeltas[d.Quad.Predicate] += countDelta if d.Quad.Label != nil { nodeDeltas[d.Quad.Label] += countDelta } nodesAdded += countDelta } // Create keys and new nodes keys := make([]*datastore.Key, 0, len(nodeDeltas)) tempNodes := make([]NodeEntry, 0, len(nodeDeltas)) for k, v := range nodeDeltas { keys = append(keys, qs.createKeyForNode(k)) tempNodes = append(tempNodes, NodeEntry{k.String(), v}) } // In accordance with the appengine datastore spec, cross group transactions // like these can only be done in batches of 5 for i := 0; i < len(nodeDeltas); i += 5 { j := int(math.Min(float64(len(nodeDeltas)-i), 5)) foundNodes := make([]NodeEntry, j) err := datastore.RunInTransaction(qs.context, func(c context.Context) error { err := datastore.GetMulti(c, keys[i:i+j], foundNodes) // Sift through for errors if me, ok := err.(appengine.MultiError); ok { for _, merr := range me { if merr != nil && merr != datastore.ErrNoSuchEntity { clog.Errorf("Error: %v", merr) return merr } } } // Carry forward the sizes of the nodes from the datastore for k := range foundNodes { if foundNodes[k].Name != "" { tempNodes[i+k].Size += foundNodes[k].Size } } _, err = datastore.PutMulti(c, keys[i:i+j], tempNodes[i:i+j]) return err }, &datastore.TransactionOptions{XG: true}) if err != nil { clog.Errorf("Error: %v", err) return 0, err } } return nodesAdded, nil } func (qs *QuadStore) updateQuads(in []graph.Delta, ids []int64) (int64, error) { keys := make([]*datastore.Key, 0, len(in)) for _, d := range in { keys = append(keys, qs.createKeyForQuad(d.Quad)) } var quadCount int64 for i := 0; i < len(in); i += 5 { // Find the closest batch of 5 j := int(math.Min(float64(len(in)-i), 5)) err := datastore.RunInTransaction(qs.context, func(c context.Context) error { foundQuads := make([]QuadEntry, j) // We don't process errors from GetMulti as they don't mean anything, // we've handled existing quad conflicts above and we overwrite everything again anyways datastore.GetMulti(c, keys, foundQuads) for k := range foundQuads { x := i + k foundQuads[k].Hash = keys[x].StringID() foundQuads[k].Subject = in[x].Quad.Subject.String() foundQuads[k].Predicate = in[x].Quad.Predicate.String() foundQuads[k].Object = in[x].Quad.Object.String() foundQuads[k].Label = quad.StringOf(in[x].Quad.Label) // If the quad exists the Added[] will be non-empty if in[x].Action == graph.Add { foundQuads[k].Added = append(foundQuads[k].Added, ids[x]) quadCount++ } else { foundQuads[k].Deleted = append(foundQuads[k].Deleted, ids[x]) quadCount-- } } _, err := datastore.PutMulti(c, keys[i:i+j], foundQuads) return err }, &datastore.TransactionOptions{XG: true}) if err != nil { return 0, err } } return quadCount, nil } func (qs *QuadStore) updateMetadata(quadsAdded int64, nodesAdded int64) error { key := qs.createKeyForMetadata() foundMetadata := new(MetadataEntry) err := datastore.RunInTransaction(qs.context, func(c context.Context) error { err := datastore.Get(c, key, foundMetadata) if err != nil && err != datastore.ErrNoSuchEntity { clog.Errorf("Error: %v", err) return err } foundMetadata.QuadCount += quadsAdded foundMetadata.NodeCount += nodesAdded _, err = datastore.Put(c, key, foundMetadata) if err != nil { clog.Errorf("Error: %v", err) } return err }, nil) return err } func (qs *QuadStore) updateLog(in []graph.Delta) ([]int64, error) { if qs.context == nil { err := errors.New("Error updating log, context is nil, graph not correctly initialised") return nil, err } if len(in) == 0 { return nil, errors.New("Nothing to log") } logEntries := make([]LogEntry, 0, len(in)) logKeys := make([]*datastore.Key, 0, len(in)) for _, d := range in { var action string if d.Action == graph.Add { action = "Add" } else { action = "Delete" } entry := LogEntry{ Action: action, Key: qs.createKeyForQuad(d.Quad).String(), Timestamp: time.Now().UnixNano(), } logEntries = append(logEntries, entry) logKeys = append(logKeys, qs.createKeyForLog()) } ids, err := datastore.PutMulti(qs.context, logKeys, logEntries) if err != nil { clog.Errorf("Error updating log: %v", err) return nil, err } out := make([]int64, 0, len(ids)) for _, id := range ids { out = append(out, id.IntID()) } return out, nil } func (qs *QuadStore) QuadIterator(dir quad.Direction, v graph.Ref) iterator.Shape { return qs.newIterator(quadKind, dir, v) } func (qs *QuadStore) NodesAllIterator() iterator.Shape { return qs.newAllIterator(nodeKind) } func (qs *QuadStore) QuadsAllIterator() iterator.Shape { return qs.newAllIterator(quadKind) } func (qs *QuadStore) ValueOf(s quad.Value) (graph.Ref, error) { id := hashOf(s) return &Token{Kind: nodeKind, Hash: id}, nil } func (qs *QuadStore) NameOf(val graph.Ref) (quad.Value, error) { if qs.context == nil { return nil, errors.New("context is nil, graph is not initialized") } else if v, ok := val.(refs.PreFetchedValue); ok { return v.NameOf(), nil } var key *datastore.Key if t, ok := val.(*Token); ok && t.Kind == nodeKind { key = qs.createKeyFromToken(t) } else { return nil, errors.New("token not valid") } // TODO (panamafrancis) implement a cache node := new(NodeEntry) err := datastore.Get(qs.context, key, node) if err != nil { return nil, err } return quad.Raw(node.Name), nil } func (qs *QuadStore) Quad(val graph.Ref) (quad.Quad, error) { if qs.context == nil { return quad.Quad{}, errors.New("context is nil, graph is not correctly initialized") } var key *datastore.Key if t, ok := val.(*Token); ok && t.Kind == quadKind { key = qs.createKeyFromToken(t) } else { return quad.Quad{}, errors.New("gae quad: token not valid") } q := new(QuadEntry) err := datastore.Get(qs.context, key, q) if err != nil { // Red herring error : ErrFieldMismatch can happen when a quad exists but a field is empty if _, ok := err.(*datastore.ErrFieldMismatch); !ok { return quad.Quad{}, err } } var label interface{} if q.Label != "" { label = q.Label } return quad.Make( q.Subject, q.Predicate, q.Object, label, ), nil } func (qs *QuadStore) Stats(ctx context.Context, exact bool) (graph.Stats, error) { if qs.context == nil { return graph.Stats{}, errors.New("error fetching size, context is nil, graph not correctly initialised") } key := qs.createKeyForMetadata() m := new(MetadataEntry) err := datastore.Get(qs.context, key, m) if err != nil { return graph.Stats{}, err } return graph.Stats{ Nodes: refs.Size{ Value: m.NodeCount, Exact: true, }, Quads: refs.Size{ Value: m.QuadCount, Exact: true, }, }, nil } func (qs *QuadStore) QuadIteratorSize(ctx context.Context, d quad.Direction, val graph.Ref) (refs.Size, error) { t, ok := val.(*Token) if !ok || t.Kind != nodeKind { return refs.Size{Value: 0, Exact: true}, nil } else if qs.context == nil { return refs.Size{}, errors.New("cannot count iterator without a valid context") } key := qs.createKeyFromToken(t) n := new(NodeEntry) err := datastore.Get(qs.context, key, n) if err != nil && err != datastore.ErrNoSuchEntity { return refs.Size{}, err } return refs.Size{Value: n.Size, Exact: true}, nil } func (qs *QuadStore) Close() error { qs.context = nil return nil } func (qs *QuadStore) QuadDirection(val graph.Ref, dir quad.Direction) (graph.Ref, error) { t, ok := val.(*Token) if !ok { return nil, errors.New("token not valid") } if t.Kind == nodeKind { return nil, errors.New("node tokens not valid") } var offset int switch dir { case quad.Subject: offset = 0 case quad.Predicate: offset = (quad.HashSize * 2) case quad.Object: offset = (quad.HashSize * 2) * 2 case quad.Label: offset = (quad.HashSize * 2) * 3 } sub := t.Hash[offset : offset+(quad.HashSize*2)] return &Token{Kind: nodeKind, Hash: sub}, nil } ================================================ FILE: graph/gaedatastore/quadstore_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build appengine appenginevm package gaedatastore import ( "errors" "net/http" "testing" "time" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest" "github.com/cayleygraph/cayley/graph/graphtest/testutil" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" "google.golang.org/appengine/aetest" ) // This is a simple test graph. // // +---+ +---+ // | A |------- ->| F |<-- // +---+ \------>+---+-/ +---+ \--+---+ // ------>|#B#| | | E | // +---+-------/ >+---+ | +---+ // | C | / v // +---+ -/ +---+ // ---- +---+/ |#G#| // \-->|#D#|------------->+---+ // +---+ // var simpleGraph = graphtest.MakeQuadSet() var simpleGraphUpdate = []quad.Quad{ quad.MakeRaw("A", "follows", "B", ""), quad.MakeRaw("F", "follows", "B", ""), quad.MakeRaw("C", "follows", "D", ""), quad.MakeRaw("X", "follows", "B", ""), } type pair struct { query string value int64 } func createInstance() (aetest.Instance, *http.Request, error) { inst, err := aetest.NewInstance(&aetest.Options{ AppID: "", StronglyConsistentDatastore: true, StartupTimeout: 15 * time.Second, }) if err != nil { return nil, nil, errors.New("Creation of new instance failed") } req1, err := inst.NewRequest("POST", "/api/v1/write", nil) if err != nil { return nil, nil, errors.New("Creation of new request failed") } return inst, req1, nil } func makeGAE(t testing.TB) (graph.QuadStore, graph.Options, func()) { inst, r, err := createInstance() require.NoError(t, err) qs, err := newQuadStore("", nil) if err != nil { inst.Close() t.Fatal(err) } qs, err = qs.(*QuadStore).ForRequest(r) if err != nil { inst.Close() t.Fatal(err) } return qs, nil, func() { qs.Close() inst.Close() } } func TestGAEAll(t *testing.T) { graphtest.TestAll(t, makeGAE, &graphtest.Config{ NoPrimitives: true, UnTyped: true, }) } func TestIterators(t *testing.T) { qs, opts, closer := makeGAE(t) defer closer() testutil.MakeWriter(t, qs, opts, graphtest.MakeQuadSet()...) require.Equal(t, int64(11), qs.Size(), "Incorrect number of quads") var expected = []quad.Quad{ quad.Make("C", "follows", "B", ""), quad.Make("C", "follows", "D", ""), } it := qs.QuadIterator(quad.Subject, qs.ValueOf(quad.Raw("C"))) graphtest.ExpectIteratedQuads(t, qs, it, expected, false) // Test contains it = qs.QuadIterator(quad.Label, qs.ValueOf(quad.Raw("status_graph"))) gqs := qs.(*QuadStore) key := gqs.createKeyForQuad(quad.Make("G", "status", "cool", "status_graph")) token := &Token{quadKind, key.StringID()} require.True(t, it.Contains(token), "Contains failed") } ================================================ FILE: graph/graphmock/graphmock.go ================================================ package graphmock import ( "context" "strconv" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) var ( _ graph.Ref = IntVal(0) _ graph.Ref = StringNode("") ) type IntVal int func (v IntVal) Key() interface{} { return v } type StringNode string func (s StringNode) Key() interface{} { return s } // Oldstore is a mocked version of the QuadStore interface, for use in tests. type Oldstore struct { Parse bool Data []string Iter iterator.Shape } func (qs *Oldstore) valueAt(i int) quad.Value { if !qs.Parse { return quad.Raw(qs.Data[i]) } iv, err := strconv.Atoi(qs.Data[i]) if err == nil { return quad.Int(iv) } return quad.String(qs.Data[i]) } func (qs *Oldstore) ValueOf(s quad.Value) (graph.Ref, error) { if s == nil { return nil, nil } for i := range qs.Data { if va := qs.valueAt(i); va != nil && s.String() == va.String() { return iterator.Int64Node(i), nil } } return nil, nil } func (qs *Oldstore) NewQuadWriter() (quad.WriteCloser, error) { return nopWriter{}, nil } type nopWriter struct{} func (nopWriter) WriteQuad(q quad.Quad) error { return nil } func (nopWriter) WriteQuads(buf []quad.Quad) (int, error) { return len(buf), nil } func (nopWriter) Close() error { return nil } func (qs *Oldstore) ApplyDeltas([]graph.Delta, graph.IgnoreOpts) error { return nil } func (qs *Oldstore) Quad(graph.Ref) quad.Quad { return quad.Quad{} } func (qs *Oldstore) QuadIterator(d quad.Direction, i graph.Ref) iterator.Shape { return qs.Iter } func (qs *Oldstore) QuadIteratorSize(ctx context.Context, d quad.Direction, val graph.Ref) (refs.Size, error) { st, err := qs.Iter.Stats(ctx) return st.Size, err } func (qs *Oldstore) NodesAllIterator() iterator.Shape { return &iterator.Null{} } func (qs *Oldstore) QuadsAllIterator() iterator.Shape { return &iterator.Null{} } func (qs *Oldstore) NameOf(v graph.Ref) (quad.Value, error) { switch v := v.(type) { case iterator.Int64Node: i := int(v) if i < 0 || i >= len(qs.Data) { return nil, nil } return qs.valueAt(i), nil case StringNode: if qs.Parse { return quad.String(v), nil } return quad.Raw(string(v)), nil default: return nil, nil } } func (qs *Oldstore) Size() int64 { return 0 } func (qs *Oldstore) DebugPrint() {} func (qs *Oldstore) OptimizeIterator(it iterator.Shape) (iterator.Shape, bool) { return iterator.NewNull(), false } func (qs *Oldstore) Close() error { return nil } func (qs *Oldstore) QuadDirection(graph.Ref, quad.Direction) graph.Ref { return iterator.Int64Node(0) } func (qs *Oldstore) RemoveQuad(t quad.Quad) {} func (qs *Oldstore) Type() string { return "oldmockstore" } type Store struct { Data []quad.Quad } var _ graph.QuadStore = &Store{} func (qs *Store) ValueOf(s quad.Value) (graph.Ref, error) { for _, q := range qs.Data { if q.Subject == s || q.Object == s { return refs.PreFetched(s), nil } } return nil, nil } func (qs *Store) ApplyDeltas([]graph.Delta, graph.IgnoreOpts) error { return nil } func (qs *Store) NewQuadWriter() (quad.WriteCloser, error) { return nopWriter{}, nil } type quadValue struct { q quad.Quad } func (q quadValue) Key() interface{} { return q.q.String() } func (qs *Store) Quad(v graph.Ref) (quad.Quad, error) { return v.(quadValue).q, nil } func (qs *Store) NameOf(v graph.Ref) (quad.Value, error) { if v == nil { return nil, nil } return v.(refs.PreFetchedValue).NameOf(), nil } func (qs *Store) RemoveQuad(t quad.Quad) {} func (qs *Store) Type() string { return "mockstore" } func (qs *Store) QuadDirection(v graph.Ref, d quad.Direction) (graph.Ref, error) { qv, err := qs.Quad(v) if err != nil { return nil, err } return refs.PreFetched(qv.Get(d)), nil } func (qs *Store) Close() error { return nil } func (qs *Store) DebugPrint() {} func (qs *Store) QuadIterator(d quad.Direction, i graph.Ref) iterator.Shape { fixed := iterator.NewFixed() v := i.(refs.PreFetchedValue).NameOf() for _, q := range qs.Data { if q.Get(d) == v { fixed.Add(quadValue{q}) } } return fixed } func (qs *Store) QuadIteratorSize(ctx context.Context, d quad.Direction, val graph.Ref) (refs.Size, error) { v := val.(refs.PreFetchedValue).NameOf() sz := refs.Size{Exact: true} for _, q := range qs.Data { if q.Get(d) == v { sz.Value++ } } return sz, nil } func (qs *Store) NodesAllIterator() iterator.Shape { set := make(map[string]bool) for _, q := range qs.Data { for _, d := range quad.Directions { n, err := qs.NameOf(refs.PreFetched(q.Get(d))) if err != nil { return iterator.NewError(err) } if n != nil { set[n.String()] = true } } } fixed := iterator.NewFixed() for k := range set { fixed.Add(refs.PreFetched(quad.Raw(k))) } return fixed } func (qs *Store) QuadsAllIterator() iterator.Shape { fixed := iterator.NewFixed() for _, q := range qs.Data { fixed.Add(quadValue{q}) } return fixed } func (qs *Store) Stats(ctx context.Context, exact bool) (graph.Stats, error) { set := make(map[string]struct{}) for _, q := range qs.Data { for _, d := range quad.Directions { n, err := qs.NameOf(refs.PreFetched(q.Get(d))) if err != nil { return graph.Stats{}, err } if n != nil { set[n.String()] = struct{}{} } } } return graph.Stats{ Nodes: refs.Size{Value: int64(len(set)), Exact: true}, Quads: refs.Size{Value: int64(len(qs.Data)), Exact: true}, }, nil } ================================================ FILE: graph/graphtest/graphtest.go ================================================ package graphtest import ( "context" "fmt" "math" "sort" "testing" "time" "github.com/cayleygraph/quad" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest/testutil" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query/path/pathtest" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/cayley/writer" ) type Config struct { NoPrimitives bool UnTyped bool // converts all values to Raw representation TimeInMs bool TimeInMcs bool TimeRound bool PageSize int // result page size for pagination (large iterator) tests OptimizesComparison bool AlwaysRunIntegration bool // always run integration tests SkipDeletedFromIterator bool SkipSizeCheckAfterDelete bool } var graphTests = []struct { name string test func(t testing.TB, gen testutil.DatabaseFunc, conf *Config) }{ {"load one quad", TestLoadOneQuad}, {"load dup", TestLoadDup}, {"load dup single", TestLoadDupSingle}, {"load dup raw", TestLoadDupRaw}, {"delete quad", TestDeleteQuad}, {"sizes", TestSizes}, {"iterator", TestIterator}, {"hasa", TestHasA}, {"set iterator", TestSetIterator}, {"deleted from iterator", TestDeletedFromIterator}, {"load typed quad", TestLoadTypedQuads}, {"add and remove", TestAddRemove}, {"node delete", TestNodeDelete}, {"iterators and next result order", TestIteratorsAndNextResultOrderA}, {"compare typed values", TestCompareTypedValues}, {"schema", TestSchema}, {"delete reinserted", TestDeleteReinserted}, {"delete reinserted dup", TestDeleteReinsertedDup}, } func TestAll(t *testing.T, gen testutil.DatabaseFunc, conf *Config) { if conf == nil { conf = &Config{} } for _, gt := range graphTests { t.Run(gt.name, func(t *testing.T) { gt.test(t, gen, conf) }) } t.Run("writers", func(t *testing.T) { TestWriters(t, gen, conf) }) t.Run("1k", func(t *testing.T) { t.Run("tx", func(t *testing.T) { Test1K(t, gen, conf) }) t.Run("batch", func(t *testing.T) { Test1KBatch(t, gen, conf) }) }) t.Run("paths", func(t *testing.T) { pathtest.RunTestMorphisms(t, gen) }) t.Run("integration", func(t *testing.T) { TestIntegration(t, gen, conf.AlwaysRunIntegration) }) } func BenchmarkAll(b *testing.B, gen testutil.DatabaseFunc, conf *Config) { b.Run("import", func(b *testing.B) { BenchmarkImport(b, gen) }) b.Run("integration", func(b *testing.B) { BenchmarkIntegration(b, gen, conf.AlwaysRunIntegration) }) } // MakeQuadSet makes a simple test graph. // // +---+ +---+ // | A |------- ->| F |<-- // +---+ \------>+---+-/ +---+ \--+---+ // ------>|#B#| | | E | // +---+-------/ >+---+ | +---+ // | C | / v // +---+ -/ +---+ // ---- +---+/ |#G#| // \-->|#D#|------------->+---+ // +---+ func MakeQuadSet() []quad.Quad { return []quad.Quad{ quad.Make("A", "follows", "B", nil), quad.Make("C", "follows", "B", nil), quad.Make("C", "follows", "D", nil), quad.Make("D", "follows", "B", nil), quad.Make("B", "follows", "F", nil), quad.Make("F", "follows", "G", nil), quad.Make("D", "follows", "G", nil), quad.Make("E", "follows", "F", nil), quad.Make("B", "status", "cool", "status_graph"), quad.Make("D", "status", "cool", "status_graph"), quad.Make("G", "status", "cool", "status_graph"), } } func IteratedQuads(t testing.TB, qs graph.QuadStore, s iterator.Shape) []quad.Quad { it := s.Iterate() defer it.Close() return IteratedQuadsNext(t, qs, it) } func IteratedQuadsNext(t testing.TB, qs graph.QuadStore, it iterator.Scanner) []quad.Quad { ctx := context.TODO() var res quad.ByQuadString for it.Next(ctx) { qsq, err := qs.Quad(it.Result()) require.NoError(t, err) res = append(res, qsq) } require.Nil(t, it.Err()) sort.Sort(res) if res == nil { return []quad.Quad(nil) // GopherJS seems to have a bug with this type conversion for a nil value } return res } func ExpectIteratedQuads(t testing.TB, qs graph.QuadStore, it iterator.Shape, exp []quad.Quad, sortQuads bool) { got := IteratedQuads(t, qs, it) if sortQuads { sort.Sort(quad.ByQuadString(exp)) sort.Sort(quad.ByQuadString(got)) } if len(exp) == 0 { exp = nil // GopherJS seems to have a bug with nil value } require.Equal(t, exp, got) } func ExpectIteratedRawStrings(t testing.TB, qs graph.QuadStore, it iterator.Shape, exp []string) { //sort.Strings(exp) got := IteratedStrings(t, qs, it) //sort.Strings(got) require.Equal(t, exp, got) } func ExpectIteratedValues(t testing.TB, qs graph.QuadStore, it iterator.Shape, exp []quad.Value, sortVals bool) { //sort.Strings(exp) got := IteratedValues(t, qs, it) //sort.Strings(got) if sortVals { exp = append([]quad.Value{}, exp...) sort.Sort(quad.ByValueString(exp)) } require.Equal(t, len(exp), len(got), "%v\nvs\n%v", exp, got) for i := range exp { if eq, ok := exp[i].(quad.Equaler); ok { require.True(t, eq.Equal(got[i])) } else { require.True(t, exp[i] == got[i], "%v\nvs\n%v\n\n%v\nvs\n%v", exp[i], got[i], exp, got) } } } func IteratedStrings(t testing.TB, qs graph.QuadStore, s iterator.Shape) []string { ctx := context.TODO() it := s.Iterate() defer it.Close() var res []string for it.Next(ctx) { qsn, err := qs.NameOf(it.Result()) require.Nil(t, err) res = append(res, quad.ToString(qsn)) } require.Nil(t, it.Err()) sort.Strings(res) return res } func IteratedValues(t testing.TB, qs graph.QuadStore, s iterator.Shape) []quad.Value { ctx := context.TODO() it := s.Iterate() var res []quad.Value for it.Next(ctx) { itn, err := qs.NameOf(it.Result()) require.Nil(t, err) res = append(res, itn) } require.Nil(t, it.Err()) sort.Sort(quad.ByValueString(res)) return res } func TestLoadOneQuad(t testing.TB, gen testutil.DatabaseFunc, c *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts) q := quad.Make( "Something", "points_to", "Something Else", "context", ) err := w.AddQuad(q) require.NoError(t, err) for _, pq := range []quad.String{"Something", "points_to", "Something Else", "context"} { tok, err := qs.ValueOf(pq) require.Nil(t, err) require.NotNil(t, tok, "quad store failed to find value: %q", pq) val, err := qs.NameOf(tok) require.Nil(t, err) require.NotNil(t, val, "quad store failed to decode value: %q", pq) require.Equal(t, pq, val, "quad store failed to roundtrip value: %q", pq) } exp := graph.Stats{ Nodes: refs.Size{Value: 4, Exact: true}, Quads: refs.Size{Value: 1, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), []quad.Quad{q}, false) } func testLoadDup(t testing.TB, gen testutil.DatabaseFunc, c *Config, single bool) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts) q := quad.Make( "Something", "points_to", "Something Else", "context", ) if single { err := w.AddQuadSet([]quad.Quad{q, q}) require.NoError(t, err) } else { err := w.AddQuad(q) require.NoError(t, err) err = w.AddQuad(q) require.NoError(t, err) } exp := graph.Stats{ Nodes: refs.Size{Value: 4, Exact: true}, Quads: refs.Size{Value: 1, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), []quad.Quad{q}, false) } func TestLoadDup(t testing.TB, gen testutil.DatabaseFunc, c *Config) { testLoadDup(t, gen, c, false) } func TestLoadDupSingle(t testing.TB, gen testutil.DatabaseFunc, c *Config) { testLoadDup(t, gen, c, true) } func TestLoadDupRaw(t testing.TB, gen testutil.DatabaseFunc, c *Config) { qs, _ := gen(t) q := quad.Make( "Something", "points_to", "Something Else", "context", ) err := qs.ApplyDeltas([]graph.Delta{ {Quad: q, Action: graph.Add}, {Quad: q, Action: graph.Add}, }, graph.IgnoreOpts{IgnoreDup: true}) require.NoError(t, err) exp := graph.Stats{ Nodes: refs.Size{Value: 4, Exact: true}, Quads: refs.Size{Value: 1, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), []quad.Quad{q}, false) } func TestWriters(t *testing.T, gen testutil.DatabaseFunc, c *Config) { t.Run("batch", func(t *testing.T) { qs, _ := gen(t) w, err := qs.NewQuadWriter() require.NoError(t, err) defer w.Close() quads := MakeQuadSet() q1 := quads[:len(quads)/2] q2 := quads[len(q1):] n, err := w.WriteQuads(q1) require.NoError(t, err) require.Equal(t, len(q1), n) n, err = w.WriteQuads(q2) require.NoError(t, err) require.Equal(t, len(q2), n) err = w.Close() require.NoError(t, err) ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), quads, true) }) for _, mis := range []bool{false, true} { for _, dup := range []bool{false, true} { name := []byte("__") if dup { name[0] = 'd' } if mis { name[1] = 'm' } t.Run(string(name), func(t *testing.T) { qs, _ := gen(t) w, err := writer.NewSingle(qs, graph.IgnoreOpts{ IgnoreDup: dup, IgnoreMissing: mis, }) require.NoError(t, err) quads := func(arr ...quad.Quad) { ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), arr, false) } deltaErr := func(exp, err error) { if exp == graph.ErrQuadNotExist && mis { require.NoError(t, err) return } else if exp == graph.ErrQuadExists && dup { require.NoError(t, err) return } e, ok := err.(*graph.DeltaError) require.True(t, ok, "expected delta error, got: %T (%v)", err, err) require.Equal(t, exp, e.Err) } // add one quad q := quad.Make("a", "b", "c", nil) err = w.AddQuad(q) require.NoError(t, err) quads(q) // try to add the same quad again err = w.AddQuad(q) deltaErr(graph.ErrQuadExists, err) quads(q) // remove quad with non-existent node err = w.RemoveQuad(quad.Make("a", "b", "not-existent", nil)) deltaErr(graph.ErrQuadNotExist, err) // remove non-existent quads err = w.RemoveQuad(quad.Make("a", "c", "b", nil)) deltaErr(graph.ErrQuadNotExist, err) err = w.RemoveQuad(quad.Make("c", "b", "a", nil)) deltaErr(graph.ErrQuadNotExist, err) // make sure store is still in correct state quads(q) // remove existing quad err = w.RemoveQuad(q) require.NoError(t, err) quads() // add the same quad again err = w.AddQuad(q) require.NoError(t, err) quads(q) }) } } } func Test1K(t *testing.T, gen testutil.DatabaseFunc, c *Config) { qs, _ := gen(t) pg := c.PageSize if pg == 0 { pg = 100 } n := pg*3 + 1 w, err := writer.NewSingle(qs, graph.IgnoreOpts{}) require.NoError(t, err) qw := graph.NewWriter(w) exp := make([]quad.Quad, 0, n) for i := 0; i < n; i++ { q := quad.Make(i, i, i, nil) exp = append(exp, q) qw.WriteQuad(q) } err = qw.Flush() require.NoError(t, err) ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), exp, true) } func Test1KBatch(t *testing.T, gen testutil.DatabaseFunc, c *Config) { qs, _ := gen(t) pg := c.PageSize if pg == 0 { pg = 100 } n := pg*3 + 1 exp := make([]quad.Quad, 0, n) for i := 0; i < n; i++ { q := quad.Make(i, i, i, nil) exp = append(exp, q) } qw, err := qs.NewQuadWriter() require.NoError(t, err) defer qw.Close() n, err = qw.WriteQuads(exp) require.NoError(t, err) require.Equal(t, len(exp), n) err = qw.Close() require.NoError(t, err) ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), exp, true) } type ValueSizer interface { SizeOf(graph.Ref) int64 } func TestSizes(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts) err := w.AddQuadSet(MakeQuadSet()) require.NoError(t, err) exp := graph.Stats{ Nodes: refs.Size{Value: 11, Exact: true}, Quads: refs.Size{Value: 11, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") if qss, ok := qs.(ValueSizer); ok { sn, err := qs.ValueOf(quad.String("B")) require.Nil(t, err) s := qss.SizeOf(sn) require.Equal(t, int64(5), s, "Unexpected quadstore value size") } err = w.RemoveQuad(quad.Make( "A", "follows", "B", nil, )) require.NoError(t, err) err = w.RemoveQuad(quad.Make( "A", "follows", "B", nil, )) require.True(t, graph.IsQuadNotExist(err)) if !conf.SkipSizeCheckAfterDelete { exp = graph.Stats{ Nodes: refs.Size{Value: 10, Exact: true}, Quads: refs.Size{Value: 10, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size after RemoveQuad") } else { exp = graph.Stats{ Nodes: refs.Size{Value: 10, Exact: true}, Quads: refs.Size{Value: 11, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") } if qss, ok := qs.(ValueSizer); ok { vn, err := qs.ValueOf(quad.String("B")) require.NoError(t, err) s := qss.SizeOf(vn) require.Equal(t, int64(4), s, "Unexpected quadstore value size") } } func TestIterator(t testing.TB, gen testutil.DatabaseFunc, _ *Config) { ctx := context.TODO() qs, opts := gen(t) testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) var it iterator.Shape it = qs.NodesAllIterator() require.NotNil(t, it) st, _ := it.Stats(ctx) size := st.Size.Value require.True(t, size > 0 && size < 23, "Unexpected size: %v", size) optIt, changed := it.Optimize(ctx) require.True(t, !changed && optIt == it, "Optimize unexpectedly changed iterator: %v, %T(%p) vs %T(%p)", changed, optIt, optIt, it, it) expect := []string{ "A", "B", "C", "D", "E", "F", "G", "follows", "status", "cool", "status_graph", } sort.Strings(expect) for i := 0; i < 2; i++ { got := IteratedStrings(t, qs, it) sort.Strings(got) require.Equal(t, expect, got, "Unexpected iterated result on repeat %d", i) } itc := it.Lookup() defer itc.Close() for _, pq := range expect { qsv, err := qs.ValueOf(quad.Raw(pq)) require.NoError(t, err) ok := itc.Contains(ctx, qsv) require.NoError(t, itc.Err()) require.True(t, ok, "Failed to find and check %q correctly", pq) } // FIXME(kortschak) Why does this fail? /* for _, pq := range []string{"baller"} { if it.Contains(qs.ValueOf(pq)) { t.Errorf("Failed to check %q correctly", pq) } } */ it = qs.QuadsAllIterator() optIt, changed = it.Optimize(ctx) require.True(t, !changed && optIt == it, "Optimize unexpectedly changed iterator: %v, %T", changed, optIt) itn := it.Iterate() defer itn.Close() require.True(t, itn.Next(ctx)) q, err := qs.Quad(itn.Result()) require.NoError(t, err) require.Nil(t, itn.Err()) require.True(t, q.IsValid(), "Invalid quad returned: %q", q) set := MakeQuadSet() var ok bool for _, e := range set { if e.String() == q.String() { ok = true break } } require.True(t, ok, "Failed to find %q during iteration, got:%q", q, set) } func TestHasA(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { qs, opts := gen(t) testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) var it iterator.Shape = graph.NewHasA(qs, graph.NewLinksTo(qs, qs.NodesAllIterator(), quad.Predicate), quad.Predicate) it, _ = it.Optimize(context.TODO()) var exp []quad.Value for i := 0; i < 8; i++ { exp = append(exp, quad.Raw("follows")) } for i := 0; i < 3; i++ { exp = append(exp, quad.Raw("status")) } ExpectIteratedValues(t, qs, it, exp, false) } func TestSetIterator(t testing.TB, gen testutil.DatabaseFunc, _ *Config) { qs, opts := gen(t) testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) expectIteratedQuads := func(it iterator.Shape, exp []quad.Quad) { ExpectIteratedQuads(t, qs, it, exp, false) } // Subject iterator. qsv, err := qs.ValueOf(quad.String("C")) require.NoError(t, err) it := qs.QuadIterator(quad.Subject, qsv) expectIteratedQuads(it, []quad.Quad{ quad.Make("C", "follows", "B", nil), quad.Make("C", "follows", "D", nil), }) and := iterator.NewAnd( qs.QuadsAllIterator(), it, ) expectIteratedQuads(and, []quad.Quad{ quad.Make("C", "follows", "B", nil), quad.Make("C", "follows", "D", nil), }) // Object iterator. qsv, err = qs.ValueOf(quad.String("F")) require.NoError(t, err) it = qs.QuadIterator(quad.Object, qsv) expectIteratedQuads(it, []quad.Quad{ quad.Make("B", "follows", "F", nil), quad.Make("E", "follows", "F", nil), }) qsv, err = qs.ValueOf(quad.String("B")) require.NoError(t, err) and = iterator.NewAnd( qs.QuadIterator(quad.Subject, qsv), it, ) expectIteratedQuads(and, []quad.Quad{ quad.Make("B", "follows", "F", nil), }) // Predicate iterator. qsv, err = qs.ValueOf(quad.String("status")) require.NoError(t, err) it = qs.QuadIterator(quad.Predicate, qsv) expectIteratedQuads(it, []quad.Quad{ quad.Make("B", "status", "cool", "status_graph"), quad.Make("D", "status", "cool", "status_graph"), quad.Make("G", "status", "cool", "status_graph"), }) // Label iterator. qsv, err = qs.ValueOf(quad.String("status_graph")) require.NoError(t, err) it = qs.QuadIterator(quad.Label, qsv) expectIteratedQuads(it, []quad.Quad{ quad.Make("B", "status", "cool", "status_graph"), quad.Make("D", "status", "cool", "status_graph"), quad.Make("G", "status", "cool", "status_graph"), }) // Order is important qsv, err = qs.ValueOf(quad.String("B")) require.NoError(t, err) and = iterator.NewAnd( qs.QuadIterator(quad.Subject, qsv), it, ) expectIteratedQuads(and, []quad.Quad{ quad.Make("B", "status", "cool", "status_graph"), }) // Order is important qsv, err = qs.ValueOf(quad.String("B")) require.NoError(t, err) and = iterator.NewAnd( it, qs.QuadIterator(quad.Subject, qsv), ) expectIteratedQuads(and, []quad.Quad{ quad.Make("B", "status", "cool", "status_graph"), }) } func TestDeleteQuad(t testing.TB, gen testutil.DatabaseFunc, _ *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) vn, err := qs.ValueOf(quad.Raw("E")) require.NoError(t, err) require.NotNil(t, vn) it := qs.QuadIterator(quad.Subject, vn) ExpectIteratedQuads(t, qs, it, []quad.Quad{ quad.Make("E", "follows", "F", nil), }, false) err = w.RemoveQuad(quad.Make("E", "follows", "F", nil)) require.NoError(t, err) qsv, err := qs.ValueOf(quad.Raw("E")) require.NoError(t, err) it = qs.QuadIterator(quad.Subject, qsv) ExpectIteratedQuads(t, qs, it, nil, false) it = qs.QuadsAllIterator() ExpectIteratedQuads(t, qs, it, []quad.Quad{ quad.Make("A", "follows", "B", nil), quad.Make("C", "follows", "B", nil), quad.Make("C", "follows", "D", nil), quad.Make("D", "follows", "B", nil), quad.Make("B", "follows", "F", nil), quad.Make("F", "follows", "G", nil), quad.Make("D", "follows", "G", nil), quad.Make("B", "status", "cool", "status_graph"), quad.Make("D", "status", "cool", "status_graph"), quad.Make("G", "status", "cool", "status_graph"), }, true) } func TestDeletedFromIterator(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { if conf.SkipDeletedFromIterator { t.SkipNow() } qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) // Subject iterator. qsv, err := qs.ValueOf(quad.Raw("E")) require.NoError(t, err) it := qs.QuadIterator(quad.Subject, qsv) ExpectIteratedQuads(t, qs, it, []quad.Quad{ quad.Make("E", "follows", "F", nil), }, false) w.RemoveQuad(quad.Make("E", "follows", "F", nil)) ExpectIteratedQuads(t, qs, it, nil, false) } func TestLoadTypedQuads(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts) values := []quad.Value{ quad.BNode("A"), quad.IRI("name"), quad.String("B"), quad.IRI("graph"), quad.IRI("B"), quad.Raw(""), quad.TypedString{Value: "10", Type: "int"}, quad.LangString{Value: "value", Lang: "en"}, quad.Int(-123456789), quad.Float(-12345e-6), quad.Bool(true), quad.Time(time.Now()), } err := w.AddQuadSet([]quad.Quad{ {values[0], values[1], values[2], values[3]}, {values[4], values[5], values[6], nil}, {values[4], values[5], values[7], nil}, {values[0], values[1], values[8], nil}, {values[0], values[1], values[9], nil}, {values[0], values[1], values[10], nil}, {values[0], values[1], values[11], nil}, }) require.NoError(t, err) for _, pq := range values { qsv, err := qs.ValueOf(pq) require.NoError(t, err) got, err := qs.NameOf(qsv) require.NoError(t, err) if !conf.UnTyped { if pt, ok := pq.(quad.Time); ok { var trim int64 if conf.TimeInMcs { trim = 1000 } else if conf.TimeInMs { trim = 1000000 } if trim > 0 { tm := time.Time(pt) seconds := tm.Unix() nanos := int64(tm.Sub(time.Unix(seconds, 0))) if conf.TimeRound { nanos = (nanos/trim + ((nanos/(trim/10))%10)/5) * trim } else { nanos = (nanos / trim) * trim } pq = quad.Time(time.Unix(seconds, nanos).UTC()) } } if eq, ok := pq.(quad.Equaler); ok { assert.True(t, eq.Equal(got), "Failed to roundtrip %q (%T), got %q (%T)", pq, pq, got, got) } else { assert.Equal(t, pq, got, "Failed to roundtrip %q (%T)", pq, pq) } // check if we can get received value again (hash roundtrip) gotv, err := qs.ValueOf(got) require.NoError(t, err) got2, err := qs.NameOf(gotv) require.NoError(t, err) assert.Equal(t, got, got2, "Failed to use returned value to get it again") } else { assert.Equal(t, quad.StringOf(pq), quad.StringOf(got), "Failed to roundtrip raw %q (%T)", pq, pq) } } exp := graph.Stats{ Nodes: refs.Size{Value: 12, Exact: true}, Quads: refs.Size{Value: 7, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") } // TestAddRemove tests add and remove // TODO(dennwc): add tests to verify that QS behaves in a right way with IgnoreOptions, // returns ErrQuadExists, ErrQuadNotExists is doing rollback. func TestAddRemove(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { qs, opts := gen(t) if opts == nil { opts = make(graph.Options) } opts["ignore_duplicate"] = true w := testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) exp := graph.Stats{ Nodes: refs.Size{Value: 11, Exact: true}, Quads: refs.Size{Value: 11, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") all := qs.NodesAllIterator() expect := []string{ "A", "B", "C", "D", "E", "F", "G", "cool", "follows", "status", "status_graph", } ExpectIteratedRawStrings(t, qs, all, expect) // Add more quads, some conflicts err = w.AddQuadSet([]quad.Quad{ quad.Make("A", "follows", "B", nil), // duplicate quad.Make("F", "follows", "B", nil), quad.Make("C", "follows", "D", nil), // duplicate quad.Make("X", "follows", "B", nil), }) assert.Nil(t, err, "AddQuadSet failed") exp = graph.Stats{ Nodes: refs.Size{Value: 12, Exact: true}, Quads: refs.Size{Value: 13, Exact: true}, } st, err = qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") all = qs.NodesAllIterator() expect = []string{ "A", "B", "C", "D", "E", "F", "G", "X", "cool", "follows", "status", "status_graph", } ExpectIteratedRawStrings(t, qs, all, expect) // Remove quad toRemove := quad.Make("X", "follows", "B", nil) err = w.RemoveQuad(toRemove) require.Nil(t, err, "RemoveQuad failed") err = w.RemoveQuad(toRemove) require.True(t, graph.IsQuadNotExist(err), "expected not exists error, got: %v", err) expect = []string{ "A", "B", "C", "D", "E", "F", "G", "cool", "follows", "status", "status_graph", } all = qs.NodesAllIterator() ExpectIteratedRawStrings(t, qs, all, expect) } func TestIteratorsAndNextResultOrderA(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { ctx := context.TODO() qs, opts := gen(t) testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) exp := graph.Stats{ Nodes: refs.Size{Value: 11, Exact: true}, Quads: refs.Size{Value: 11, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") qsv, err := qs.ValueOf(quad.Raw("C")) require.NoError(t, err) fixed := iterator.NewFixed(qsv) qsv, err = qs.ValueOf(quad.Raw("follows")) require.NoError(t, err) fixed2 := iterator.NewFixed(qsv) all := qs.NodesAllIterator() const allTag = "all" innerAnd := iterator.NewAnd( graph.NewLinksTo(qs, fixed2, quad.Predicate), graph.NewLinksTo(qs, iterator.Tag(all, allTag), quad.Object), ) hasa := graph.NewHasA(qs, innerAnd, quad.Subject) outerAnd := iterator.NewAnd(fixed, hasa).Iterate() require.True(t, outerAnd.Next(ctx), "Expected one matching subtree") val := outerAnd.Result() qsn, err := qs.NameOf(val) require.NoError(t, err) require.Equal(t, quad.Raw("C"), qsn) var ( got []string expect = []string{"B", "D"} ) for { m := make(map[string]graph.Ref, 1) outerAnd.TagResults(m) qsn, err = qs.NameOf(m[allTag]) require.NoError(t, err) got = append(got, quad.ToString(qsn)) if !outerAnd.NextPath(ctx) { break } } sort.Strings(got) require.Equal(t, expect, got) require.True(t, !outerAnd.Next(ctx), "More than one possible top level output?") } const lt, lte, gt, gte = iterator.CompareLT, iterator.CompareLTE, iterator.CompareGT, iterator.CompareGTE var tzero = time.Unix(time.Now().Unix(), 0) var casesCompare = []struct { op iterator.Operator val quad.Value expect []quad.Value }{ {lt, quad.BNode("b"), []quad.Value{ quad.BNode("alice"), }}, {lte, quad.BNode("bob"), []quad.Value{ quad.BNode("alice"), quad.BNode("bob"), }}, {lt, quad.String("b"), []quad.Value{ quad.String("alice"), }}, {lte, quad.String("bob"), []quad.Value{ quad.String("alice"), quad.String("bob"), }}, {gte, quad.String("b"), []quad.Value{ quad.String("bob"), quad.String("charlie"), quad.String("dani"), }}, {lt, quad.IRI("b"), []quad.Value{ quad.IRI("alice"), }}, {lte, quad.IRI("bob"), []quad.Value{ quad.IRI("alice"), quad.IRI("bob"), }}, {lte, quad.IRI("bob"), []quad.Value{ quad.IRI("alice"), quad.IRI("bob"), }}, {gte, quad.Int(111), []quad.Value{ quad.Int(112), quad.Int(math.MaxInt64 - 1), quad.Int(math.MaxInt64), }}, {gte, quad.Int(110), []quad.Value{ quad.Int(110), quad.Int(112), quad.Int(math.MaxInt64 - 1), quad.Int(math.MaxInt64), }}, {lt, quad.Int(20), []quad.Value{ quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64), }}, {lte, quad.Int(20), []quad.Value{ quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64), quad.Int(20), }}, {lte, quad.Time(tzero.Add(time.Hour)), []quad.Value{ quad.Time(tzero), quad.Time(tzero.Add(time.Hour)), }}, {gt, quad.Time(tzero.Add(time.Hour)), []quad.Value{ quad.Time(tzero.Add(time.Hour * 49)), quad.Time(tzero.Add(time.Hour * 24 * 365)), }}, // precision tests {gt, quad.Int(math.MaxInt64 - 1), []quad.Value{ quad.Int(math.MaxInt64), }}, {gte, quad.Int(math.MaxInt64 - 1), []quad.Value{ quad.Int(math.MaxInt64 - 1), quad.Int(math.MaxInt64), }}, {lt, quad.Int(math.MinInt64 + 1), []quad.Value{ quad.Int(math.MinInt64), }}, {lte, quad.Int(math.MinInt64 + 1), []quad.Value{ quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64), }}, } func TestCompareTypedValues(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { if conf.UnTyped { t.SkipNow() } qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts) t1 := tzero t2 := t1.Add(time.Hour) t3 := t2.Add(time.Hour * 48) t4 := t1.Add(time.Hour * 24 * 365) quads := []quad.Quad{ {quad.BNode("alice"), quad.BNode("bob"), quad.BNode("charlie"), quad.BNode("dani")}, {quad.IRI("alice"), quad.IRI("bob"), quad.IRI("charlie"), quad.IRI("dani")}, {quad.String("alice"), quad.String("bob"), quad.String("charlie"), quad.String("dani")}, {quad.Int(100), quad.Int(112), quad.Int(110), quad.Int(20)}, {quad.Time(t1), quad.Time(t2), quad.Time(t3), quad.Time(t4)}, // test precision as well {quad.Int(math.MaxInt64), quad.Int(math.MaxInt64 - 1), quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64)}, } err := w.AddQuadSet(quads) require.NoError(t, err) var vals []quad.Value for _, q := range quads { for _, d := range quad.Directions { if v := q.Get(d); v != nil { vals = append(vals, v) } } } ExpectIteratedValues(t, qs, qs.NodesAllIterator(), vals, true) for _, c := range casesCompare { //t.Log(c.op, c.val) it := iterator.NewComparison(qs.NodesAllIterator(), c.op, c.val, qs) ExpectIteratedValues(t, qs, it, c.expect, true) } ctx := context.TODO() for _, c := range casesCompare { s := shape.Compare(shape.AllNodes{}, c.op, c.val) ns, ok := shape.Optimize(ctx, s, qs) require.Equal(t, conf.OptimizesComparison, ok) if conf.OptimizesComparison { require.NotEqual(t, s, ns) } else { require.Equal(t, s, ns) } nit := shape.BuildIterator(ctx, qs, ns) ExpectIteratedValues(t, qs, nit, c.expect, true) } } func TestNodeDelete(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) del := quad.Raw("D") err := w.RemoveNode(del) require.NoError(t, err) exp := MakeQuadSet() for i := 0; i < len(exp); i++ { for _, d := range quad.Directions { if exp[i].Get(d) == del { exp = append(exp[:i], exp[i+1:]...) i-- break } } } ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), exp, true) ExpectIteratedValues(t, qs, qs.NodesAllIterator(), []quad.Value{ quad.Raw("A"), quad.Raw("B"), quad.Raw("C"), quad.Raw("E"), quad.Raw("F"), quad.Raw("G"), quad.Raw("cool"), quad.Raw("follows"), quad.Raw("status"), quad.Raw("status_graph"), }, true) } func TestSchema(t testing.TB, gen testutil.DatabaseFunc, conf *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) type Person struct { _ struct{} `quad:"@type > ex:Person"` ID quad.IRI `quad:"@id" json:"id"` Name string `quad:"ex:name" json:"name"` Something []quad.IRI `quad:"isParentOf < *,optional" json:"something"` } p := Person{ ID: quad.IRI("ex:bob"), Name: "Bob", } sch := schema.NewConfig() qw := graph.NewWriter(w) id, err := sch.WriteAsQuads(qw, p) require.NoError(t, err) err = qw.Close() require.NoError(t, err) require.Equal(t, p.ID, id) var p2 Person err = sch.LoadTo(nil, qs, &p2, id) require.NoError(t, err) require.Equal(t, p, p2) } func TestDeleteReinserted(t testing.TB, gen testutil.DatabaseFunc, _ *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) err := w.AddQuadSet([]quad.Quad{ quad.Make("", "", "Feeling happy", nil), quad.Make("", "", "", nil), }) require.NoError(t, err, "Add quadset failed") ctx := context.TODO() q := quad.Make("", "", "", nil) for i := 0; i < 2; i++ { err = w.AddQuad(q) require.NoError(t, err, "Add quad failed") err = w.RemoveQuad(q) require.NoError(t, err, "Remove quad failed") refs, err := graph.RefsOf(ctx, qs, []quad.Value{ q.Subject, q.Predicate, q.Object, }) require.NoError(t, err, "Get values failed") require.Len(t, refs, 3) for _, r := range refs { require.NotNil(t, r) } } } func TestDeleteReinsertedDup(t testing.TB, gen testutil.DatabaseFunc, _ *Config) { qs, opts := gen(t) w := testutil.MakeWriter(t, qs, opts, MakeQuadSet()...) err := w.AddQuadSet([]quad.Quad{ quad.Make("", "", "Feeling happy", nil), quad.Make("", "", "", nil), }) require.NoError(t, err, "Add quadset failed") ctx := context.TODO() q := quad.Make("", "", "", nil) for i := 0; i < 2; i++ { err = w.AddQuad(q) require.NoError(t, err, "Add quad failed") // must be ignored err = w.AddQuad(q) require.NoError(t, err, "Add quad failed") err = w.RemoveQuad(q) require.NoError(t, err, "Remove quad failed") refs, err := graph.RefsOf(ctx, qs, []quad.Value{ q.Subject, q.Predicate, }) require.NoError(t, err, "Get values failed") require.Len(t, refs, 2) for _, r := range refs { require.NotNil(t, r) } // the node should be garbage-collected refs, err = graph.RefsOf(ctx, qs, []quad.Value{ q.Object, }) if err == nil { // FIXME(dennwc): the graphlog.SplitDeltas adds an increment even though the quad is duplicated and ignored t.Skip("value must be garbage-collected") } } } func irif(format string, args ...interface{}) quad.IRI { return quad.IRI(fmt.Sprintf(format, args...)) } func BenchmarkImport(b *testing.B, gen testutil.DatabaseFunc) { b.StopTimer() qs, _ := gen(b) w, err := qs.NewQuadWriter() require.NoError(b, err) defer w.Close() const ( mult = 10 perBatch = 100 ) quads := make([]quad.Quad, 0, mult*b.N) for i := 0; i < mult*b.N; i++ { quads = append(quads, quad.Quad{ Subject: irif("n%d", i/5), Predicate: quad.IRI("sub"), Object: irif("n%d", i/2+i%2), }) } b.ResetTimer() b.StartTimer() for len(quads) > 0 { batch := quads if len(batch) > perBatch { batch = batch[:perBatch] } n, err := w.WriteQuads(batch) if err != nil { b.Fatal(err) } else if n != len(batch) { b.Fatal(n) } quads = quads[len(batch):] } err = w.Close() if err != nil { b.Fatal(err) } b.StopTimer() } ================================================ FILE: graph/graphtest/integration.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graphtest import ( "context" "fmt" "os" "path/filepath" "reflect" "sort" "testing" "time" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest/testutil" "github.com/cayleygraph/cayley/internal" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/gizmo" _ "github.com/cayleygraph/cayley/writer" ) const ( format = "nquads" timeout = 300 * time.Second ) const ( nSpeed = "Speed" nLakeH = "The Lake House" SandraB = "Sandra Bullock" KeanuR = "Keanu Reeves" ) func checkIntegration(t testing.TB, force bool) { if testing.Short() { t.SkipNow() } if !force && os.Getenv("RUN_INTEGRATION") != "true" { t.Skip("skipping integration tests; set RUN_INTEGRATION=true to run them") } } func TestIntegration(t *testing.T, gen testutil.DatabaseFunc, force bool) { checkIntegration(t, force) qs := prepare(t, gen) checkQueries(t, qs, timeout) } func BenchmarkIntegration(t *testing.B, gen testutil.DatabaseFunc, force bool) { checkIntegration(t, force) benchmarkQueries(t, gen) } func costarTag(id, c1, c1m, c2, c2m string) map[string]string { return map[string]string{ "id": id, "costar1_actor": c1, "costar1_movie": c1m, "costar2_actor": c2, "costar2_movie": c2m, } } var queries = []struct { message string long bool query string tag string // for testing skip bool expect []interface{} }{ // Easy one to get us started. How quick is the most straightforward retrieval? { message: "name predicate", query: ` g.V("Humphrey Bogart").in("").all() `, expect: []interface{}{ map[string]string{"id": ""}, }, }, // Grunty queries. // 2014-07-12: This one seems to return in ~20ms in memory; // that's going to be measurably slower for every other backend. { message: "two large sets with no intersection", query: ` function getId(x) { return g.V(x).in("") } var actor_to_film = g.M().in("").in("") getId("Oliver Hardy").follow(actor_to_film).out("").intersect( getId("Mel Blanc").follow(actor_to_film).out("")).all() `, expect: nil, }, // 2014-07-12: This one takes about 4 whole seconds in memory. This is a behemoth. { message: "three huge sets with small intersection", long: true, query: ` function getId(x) { return g.V(x).in("") } var actor_to_film = g.M().in("").in("") var a = getId("Oliver Hardy").follow(actor_to_film).followR(actor_to_film) var b = getId("Mel Blanc").follow(actor_to_film).followR(actor_to_film) var c = getId("Billy Gilbert").follow(actor_to_film).followR(actor_to_film) seen = {} a.intersect(b).intersect(c).forEach(function (d) { if (!(d.id in seen)) { seen[d.id] = true; g.emit(d) } }) `, expect: []interface{}{ map[string]string{"id": ""}, map[string]string{"id": ""}, }, }, // This is more of an optimization problem that will get better over time. This takes a lot // of wrong turns on the walk down to what is ultimately the name, but top AND has it easy // as it has a fixed ID. Exercises Contains(). { message: "the helpless checker", long: true, query: ` g.V().as("person").in("").in().in().out("").is("Casablanca").all() `, tag: "person", expect: []interface{}{ map[string]string{"id": "Casablanca", "person": "Ingrid Bergman"}, map[string]string{"id": "Casablanca", "person": "Madeleine LeBeau"}, map[string]string{"id": "Casablanca", "person": "Joy Page"}, map[string]string{"id": "Casablanca", "person": "Claude Rains"}, map[string]string{"id": "Casablanca", "person": "S.Z. Sakall"}, map[string]string{"id": "Casablanca", "person": "Helmut Dantine"}, map[string]string{"id": "Casablanca", "person": "Conrad Veidt"}, map[string]string{"id": "Casablanca", "person": "Paul Henreid"}, map[string]string{"id": "Casablanca", "person": "Peter Lorre"}, map[string]string{"id": "Casablanca", "person": "Sydney Greenstreet"}, map[string]string{"id": "Casablanca", "person": "Leonid Kinskey"}, map[string]string{"id": "Casablanca", "person": "Lou Marcelle"}, map[string]string{"id": "Casablanca", "person": "Dooley Wilson"}, map[string]string{"id": "Casablanca", "person": "John Qualen"}, map[string]string{"id": "Casablanca", "person": "Humphrey Bogart"}, }, }, // Exercises Not().Contains(), as above. { message: "the helpless checker, negated (films without Ingrid Bergman)", long: true, query: ` g.V().as("person").in("").in().in().out("").except(g.V("Ingrid Bergman").in("").in().in().out("")).is("Casablanca").all() `, tag: "person", expect: nil, }, { message: "the helpless checker, negated (without actors Ingrid Bergman)", long: true, query: ` g.V().as("person").in("").except(g.V("Ingrid Bergman").in("")).in().in().out("").is("Casablanca").all() `, tag: "person", expect: []interface{}{ map[string]string{"id": "Casablanca", "person": "Madeleine LeBeau"}, map[string]string{"id": "Casablanca", "person": "Joy Page"}, map[string]string{"id": "Casablanca", "person": "Claude Rains"}, map[string]string{"id": "Casablanca", "person": "S.Z. Sakall"}, map[string]string{"id": "Casablanca", "person": "Helmut Dantine"}, map[string]string{"id": "Casablanca", "person": "Conrad Veidt"}, map[string]string{"id": "Casablanca", "person": "Paul Henreid"}, map[string]string{"id": "Casablanca", "person": "Peter Lorre"}, map[string]string{"id": "Casablanca", "person": "Sydney Greenstreet"}, map[string]string{"id": "Casablanca", "person": "Leonid Kinskey"}, map[string]string{"id": "Casablanca", "person": "Lou Marcelle"}, map[string]string{"id": "Casablanca", "person": "Dooley Wilson"}, map[string]string{"id": "Casablanca", "person": "John Qualen"}, map[string]string{"id": "Casablanca", "person": "Humphrey Bogart"}, }, }, //Q: Who starred in both "The Net" and "Speed" ? //A: "Sandra Bullock" { message: "Net and Speed", query: common + `m1_actors.intersect(m2_actors).out("").all() `, expect: []interface{}{ map[string]string{"id": SandraB, "movie1": "The Net", "movie2": nSpeed}, }, }, //Q: Did "Keanu Reeves" star in "The Net" ? //A: No { message: "Keanu in The Net", query: common + `actor2.intersect(m1_actors).out("").all() `, expect: nil, }, //Q: Did "Keanu Reeves" star in "Speed" ? //A: Yes { message: "Keanu in Speed", query: common + `actor2.intersect(m2_actors).out("").all() `, expect: []interface{}{ map[string]string{"id": KeanuR, "movie2": nSpeed}, }, }, //Q: Has "Keanu Reeves" co-starred with anyone who starred in "The Net" ? //A: "Keanu Reeves" was in "Speed" and "The Lake House" with "Sandra Bullock", // who was in "The Net" { message: "Keanu with other in The Net", long: true, query: common + `actor2.follow(coStars1).intersect(m1_actors).out("").all() `, expect: []interface{}{ map[string]string{"id": SandraB, "movie1": "The Net", "costar1_movie": nSpeed}, map[string]string{"movie1": "The Net", "costar1_movie": nLakeH, "id": SandraB}, }, }, //Q: Do "Keanu Reeves" and "Sandra Bullock" have any commons co-stars? //A: Yes, many. For example: SB starred with "Steve Martin" in "The Prince // of Egypt", and KR starred with Steven Martin in "Parenthood". { message: "Keanu and Bullock with other", long: true, query: common + `actor1.save("","costar1_actor").follow(coStars1).intersect(actor2.save("","costar2_actor").follow(coStars2)).out("").all() `, expect: []interface{}{ costarTag(SandraB, SandraB, "The Proposal", KeanuR, nSpeed), costarTag(SandraB, SandraB, "The Proposal", KeanuR, nLakeH), costarTag("Mary Steenburgen", SandraB, "The Proposal", KeanuR, "Parenthood"), costarTag("Craig T. Nelson", SandraB, "The Proposal", KeanuR, "The Devil's Advocate"), costarTag(SandraB, SandraB, "Crash", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Crash", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Gun Shy", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Gun Shy", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Demolition Man", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Demolition Man", KeanuR, nLakeH), costarTag("Benjamin Bratt", SandraB, "Demolition Man", KeanuR, "Thumbsucker"), costarTag(SandraB, SandraB, "Divine Secrets of the Ya-Ya Sisterhood", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Divine Secrets of the Ya-Ya Sisterhood", KeanuR, nLakeH), costarTag("Shirley Knight", SandraB, "Divine Secrets of the Ya-Ya Sisterhood", KeanuR, "The Private Lives of Pippa Lee"), costarTag(SandraB, SandraB, "A Time to Kill", KeanuR, nSpeed), costarTag(SandraB, SandraB, "A Time to Kill", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Forces of Nature", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Forces of Nature", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Hope Floats", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Hope Floats", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Infamous", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Infamous", KeanuR, nLakeH), costarTag("Jeff Daniels", SandraB, "Infamous", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Love Potion No. 9", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Love Potion No. 9", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Miss Congeniality", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Miss Congeniality", KeanuR, nLakeH), costarTag("Benjamin Bratt", SandraB, "Miss Congeniality", KeanuR, "Thumbsucker"), costarTag(SandraB, SandraB, "Miss Congeniality 2: Armed and Fabulous", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Miss Congeniality 2: Armed and Fabulous", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Murder by Numbers", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Murder by Numbers", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Practical Magic", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Practical Magic", KeanuR, nLakeH), costarTag("Dianne Wiest", SandraB, "Practical Magic", KeanuR, "Parenthood"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Flying"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Animatrix"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Tune in Tomorrow"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Last Time I Committed Suicide"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Constantine"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Permanent Record"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Dangerous Liaisons"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Private Lives of Pippa Lee"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "A Scanner Darkly"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "A Walk in the Clouds"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Hardball"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Life Under Water"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Much Ado About Nothing"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "My Own Private Idaho"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Parenthood"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Point Break"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Providence"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "River's Edge"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Something's Gotta Give"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, nSpeed), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Sweet November"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, nLakeH), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Matrix Reloaded"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Matrix Revisited"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Prince of Pennsylvania"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Replacements"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Even Cowgirls Get the Blues"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Youngblood"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Bill & Ted's Bogus Journey"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Bill & Ted's Excellent Adventure"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Johnny Mnemonic"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Devil's Advocate"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Thumbsucker"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "I Love You to Death"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Bram Stoker's Dracula"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Gift"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Little Buddha"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Night Watchman"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Chain Reaction"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "Babes in Toyland"), costarTag(KeanuR, SandraB, nSpeed, KeanuR, "The Day the Earth Stood Still"), costarTag(SandraB, SandraB, nSpeed, KeanuR, nSpeed), costarTag(SandraB, SandraB, nSpeed, KeanuR, nLakeH), costarTag("Dennis Hopper", SandraB, nSpeed, KeanuR, "River's Edge"), costarTag("Dennis Hopper", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Jeff Daniels", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Joe Morton", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Alan Ruck", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Glenn Plummer", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Carlos Carrasco", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Beth Grant", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Richard Lineback", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Hawthorne James", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Jordan Lund", SandraB, nSpeed, KeanuR, nSpeed), costarTag("Thomas Rosales, Jr.", SandraB, nSpeed, KeanuR, nSpeed), costarTag(SandraB, SandraB, "Speed 2: Cruise Control", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Speed 2: Cruise Control", KeanuR, nLakeH), costarTag("Glenn Plummer", SandraB, "Speed 2: Cruise Control", KeanuR, nSpeed), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Flying"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Animatrix"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Tune in Tomorrow"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Last Time I Committed Suicide"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Constantine"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Permanent Record"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Dangerous Liaisons"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Private Lives of Pippa Lee"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "A Scanner Darkly"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "A Walk in the Clouds"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Hardball"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Life Under Water"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Much Ado About Nothing"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "My Own Private Idaho"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Parenthood"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Point Break"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Providence"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "River's Edge"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Something's Gotta Give"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, nSpeed), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Sweet November"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, nLakeH), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Matrix Reloaded"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Matrix Revisited"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Prince of Pennsylvania"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Replacements"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Even Cowgirls Get the Blues"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Youngblood"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Bill & Ted's Bogus Journey"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Bill & Ted's Excellent Adventure"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Johnny Mnemonic"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Devil's Advocate"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Thumbsucker"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "I Love You to Death"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Bram Stoker's Dracula"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Gift"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Little Buddha"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Night Watchman"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Chain Reaction"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "Babes in Toyland"), costarTag(KeanuR, SandraB, nLakeH, KeanuR, "The Day the Earth Stood Still"), costarTag(SandraB, SandraB, nLakeH, KeanuR, nSpeed), costarTag(SandraB, SandraB, nLakeH, KeanuR, nLakeH), costarTag("Christopher Plummer", SandraB, nLakeH, KeanuR, nLakeH), costarTag("Dylan Walsh", SandraB, nLakeH, KeanuR, nLakeH), costarTag("Shohreh Aghdashloo", SandraB, nLakeH, KeanuR, nLakeH), costarTag("Lynn Collins", SandraB, nLakeH, KeanuR, nLakeH), costarTag(SandraB, SandraB, "The Net", KeanuR, nSpeed), costarTag(SandraB, SandraB, "The Net", KeanuR, nLakeH), costarTag("Michelle Pfeiffer", SandraB, "The Prince of Egypt", KeanuR, "Dangerous Liaisons"), costarTag(SandraB, SandraB, "The Prince of Egypt", KeanuR, nSpeed), costarTag(SandraB, SandraB, "The Prince of Egypt", KeanuR, nLakeH), costarTag("Steve Martin", SandraB, "The Prince of Egypt", KeanuR, "Parenthood"), costarTag(SandraB, SandraB, "Two Weeks Notice", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Two Weeks Notice", KeanuR, nLakeH), costarTag(SandraB, SandraB, "While You Were Sleeping", KeanuR, nSpeed), costarTag(SandraB, SandraB, "While You Were Sleeping", KeanuR, nLakeH), costarTag("Jack Warden", SandraB, "While You Were Sleeping", KeanuR, "The Replacements"), costarTag(SandraB, SandraB, "28 Days", KeanuR, nSpeed), costarTag(SandraB, SandraB, "28 Days", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Premonition", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Premonition", KeanuR, nLakeH), costarTag("Peter Stormare", SandraB, "Premonition", KeanuR, "Constantine"), costarTag(SandraB, SandraB, "Wrestling Ernest Hemingway", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Wrestling Ernest Hemingway", KeanuR, nLakeH), costarTag(SandraB, SandraB, "Fire on the Amazon", KeanuR, nSpeed), costarTag(SandraB, SandraB, "Fire on the Amazon", KeanuR, nLakeH), costarTag("River Phoenix", SandraB, "The Thing Called Love", KeanuR, "My Own Private Idaho"), costarTag("River Phoenix", SandraB, "The Thing Called Love", KeanuR, "I Love You to Death"), costarTag(SandraB, SandraB, "The Thing Called Love", KeanuR, nSpeed), costarTag(SandraB, SandraB, "The Thing Called Love", KeanuR, nLakeH), costarTag(SandraB, SandraB, "In Love and War", KeanuR, nSpeed), costarTag(SandraB, SandraB, "In Love and War", KeanuR, nLakeH), }, }, { message: "Save a number of predicates around a set of nodes", query: ` g.V("_:9037", "_:49278", "_:44112", "_:44709", "_:43382").save("", "char").save("", "act").saveR("", "film").all() `, expect: []interface{}{ map[string]string{"act": "", "char": "Rick Blaine", "film": "", "id": "_:9037"}, map[string]string{"act": "", "char": "Sam Spade", "film": "", "id": "_:49278"}, map[string]string{"act": "", "char": "Philip Marlowe", "film": "", "id": "_:44112"}, map[string]string{"act": "", "char": "Captain Queeg", "film": "", "id": "_:44709"}, map[string]string{"act": "", "char": "Charlie Allnut", "film": "", "id": "_:43382"}, }, }, } const common = ` var movie1 = g.V().has("", "The Net") var movie2 = g.V().has("", "Speed") var actor1 = g.V().has("", "Sandra Bullock") var actor2 = g.V().has("", "Keanu Reeves") // (film) -> starring -> (actor) var filmToActor = g.Morphism().out("").out("") // (actor) -> starring -> [film -> starring -> (actor)] var coStars1 = g.Morphism().in("").in("").save("","costar1_movie").follow(filmToActor) var coStars2 = g.Morphism().in("").in("").save("","costar2_movie").follow(filmToActor) // Stars for the movies "The Net" and "Speed" var m1_actors = movie1.save("","movie1").follow(filmToActor) var m2_actors = movie2.save("","movie2").follow(filmToActor) ` func prepare(t testing.TB, gen testutil.DatabaseFunc) graph.QuadStore { qs, _ := gen(t) const needsLoad = true // TODO: support local setup if needsLoad { qw, err := qs.NewQuadWriter() if err != nil { require.NoError(t, err) } start := time.Now() for _, p := range []string{"./", "../"} { err = internal.Load(qw, 0, filepath.Join(p, "../../data/30kmoviedata.nq.gz"), format) if err == nil || !os.IsNotExist(err) { break } } if err != nil { qw.Close() require.NoError(t, err) } err = qw.Close() if err != nil { require.NoError(t, err) } t.Logf("loaded data in %v", time.Since(start)) } return qs } func checkQueries(t *testing.T, qs graph.QuadStore, timeout time.Duration) { if qs == nil { t.Fatal("not initialized") } for _, test := range queries { t.Run(test.message, func(t *testing.T) { if testing.Short() && test.long { t.SkipNow() } if test.skip { t.SkipNow() } start := time.Now() ses := gizmo.NewSession(qs) ctx := context.Background() if timeout > 0 { var cancel func() ctx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } it, err := ses.Execute(ctx, test.query, query.Options{ Collation: query.JSON, }) if err != nil { t.Fatal(err) } defer it.Close() var got []interface{} for it.Next(ctx) { got = append(got, it.Result()) } t.Logf("%12v %v", time.Since(start), test.message) if len(got) != len(test.expect) { t.Errorf("Unexpected number of results, got:%d expect:%d on %s.", len(got), len(test.expect), test.message) return } if unsortedEqual(got, test.expect) { return } t.Errorf("Unexpected results for %s:\n", test.message) for i := range got { t.Errorf("\n\tgot:%#v\n\texpect:%#v\n", got[i], test.expect[i]) } }) } } func unsortedEqual(got, expect []interface{}) bool { gotList := convertToStringList(got) expectList := convertToStringList(expect) return reflect.DeepEqual(gotList, expectList) } func convertToStringList(in []interface{}) []string { var out []string for _, x := range in { if xc, ok := x.(map[string]string); ok { for k, v := range xc { out = append(out, fmt.Sprint(k, ":", v)) } } else { for k, v := range x.(map[string]interface{}) { out = append(out, fmt.Sprint(k, ":", v)) } } } sort.Strings(out) return out } func benchmarkQueries(b *testing.B, gen testutil.DatabaseFunc) { qs := prepare(b, gen) for _, bench := range queries { b.Run(bench.message, func(b *testing.B) { if testing.Short() && bench.long { b.Skip() } b.StopTimer() b.ResetTimer() for i := 0; i < b.N; i++ { func() { ctx := context.Background() if timeout > 0 { var cancel func() ctx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } ses := gizmo.NewSession(qs) b.StartTimer() it, err := ses.Execute(ctx, bench.query, query.Options{ Collation: query.Raw, }) if err != nil { b.Fatal(err) } defer it.Close() n := 0 for it.Next(ctx) { n++ } if err = it.Err(); err != nil { b.Fatal(err) } b.StopTimer() if n != len(bench.expect) { b.Fatalf("unexpected number of results: %d vs %d", n, len(bench.expect)) } }() } }) } } ================================================ FILE: graph/graphtest/testutil/testutil.go ================================================ package testutil import ( "os" "path/filepath" "testing" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/nquads" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/writer" ) type DatabaseFunc func(t testing.TB) (graph.QuadStore, graph.Options) func LoadGraph(t testing.TB, path string) []quad.Quad { var ( f *os.File err error ) const levels = 5 for i := 0; i < levels; i++ { f, err = os.Open(path) if i+1 < levels && os.IsNotExist(err) { path = filepath.Join("../", path) } else if err != nil { t.Fatalf("Failed to open %q: %v", path, err) } else { break } } defer f.Close() dec := nquads.NewReader(f, false) quads, err := quad.ReadAll(dec) if err != nil { t.Fatalf("Failed to Unmarshal: %v", err) } return quads } func MakeWriter(t testing.TB, qs graph.QuadStore, opts graph.Options, data ...quad.Quad) graph.QuadWriter { w, err := writer.NewSingleReplication(qs, opts) require.NoError(t, err) if len(data) > 0 { err = w.AddQuadSet(data) require.NoError(t, err) } return w } ================================================ FILE: graph/hasa.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph // Defines one of the base iterators, the HasA iterator. The HasA takes a // subiterator of links, and acts as an iterator of nodes in the given // direction. The name comes from the idea that a "link HasA subject" or a "link // HasA predicate". // // HasA is weird in that it may return the same value twice if on the Next() // path. That's okay -- in reality, it can be viewed as returning the value for // a new quad, but to make logic much simpler, here we have the HasA. // // Likewise, it's important to think about Contains()ing a HasA. When given a // value to check, it means "Check all predicates that have this value for your // direction against the subiterator." This would imply that there's more than // one possibility for the same Contains()ed value. While we could return the // number of options, it's simpler to return one, and then call NextPath() // enough times to enumerate the options. (In fact, one could argue that the // raison d'etre for NextPath() is this iterator). // // Alternatively, can be seen as the dual of the LinksTo iterator. import ( "context" "fmt" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) // A HasA consists of a reference back to the graph.QuadStore that it references, // a primary subiterator, a direction in which the quads for that subiterator point, // and a temporary holder for the iterator generated on Contains(). type HasA struct { qs QuadIndexer primary iterator.Shape dir quad.Direction } // NewHasA construct a new HasA iterator, given the quad subiterator, and the quad // direction for which it stands. func NewHasA(qs QuadIndexer, subIt iterator.Shape, d quad.Direction) *HasA { return &HasA{ qs: qs, primary: subIt, dir: d, } } func (it *HasA) Iterate() iterator.Scanner { return newHasANext(it.qs, it.primary.Iterate(), it.dir) } func (it *HasA) Lookup() iterator.Index { return newHasAContains(it.qs, it.primary.Lookup(), it.dir) } // SubIterators returns our sole subiterator. func (it *HasA) SubIterators() []iterator.Shape { return []iterator.Shape{it.primary} } // Direction accessor. func (it *HasA) Direction() quad.Direction { return it.dir } // Optimize pass the Optimize() call along to the subiterator. If it becomes Null, // then the HasA becomes Null (there are no quads that have any directions). func (it *HasA) Optimize(ctx context.Context) (iterator.Shape, bool) { newPrimary, changed := it.primary.Optimize(ctx) if changed { it.primary = newPrimary if iterator.IsNull(it.primary) { return it.primary, true } } return it, false } func (it *HasA) String() string { return fmt.Sprintf("HasA(%v)", it.dir) } // Stats returns the statistics on the HasA iterator. This is curious. Next // cost is easy, it's an extra call or so on top of the subiterator Next cost. // ContainsCost involves going to the graph.QuadStore, iterating out values, and hoping // one sticks -- potentially expensive, depending on fanout. Size, however, is // potentially smaller. we know at worst it's the size of the subiterator, but // if there are many repeated values, it could be much smaller in totality. func (it *HasA) Stats(ctx context.Context) (iterator.Costs, error) { subitStats, err := it.primary.Stats(ctx) // TODO(barakmich): These should really come from the quadstore itself // and be optimized. faninFactor := int64(1) fanoutFactor := int64(30) nextConstant := int64(2) quadConstant := int64(1) return iterator.Costs{ NextCost: quadConstant + subitStats.NextCost, ContainsCost: (fanoutFactor * nextConstant) * subitStats.ContainsCost, Size: refs.Size{ Value: faninFactor * subitStats.Size.Value, Exact: false, }, }, err } // A HasA consists of a reference back to the graph.QuadStore that it references, // a primary subiterator, a direction in which the quads for that subiterator point, // and a temporary holder for the iterator generated on Contains(). type hasANext struct { qs QuadIndexer primary iterator.Scanner dir quad.Direction result refs.Ref err error } // Construct a new HasA iterator, given the quad subiterator, and the quad // direction for which it stands. func newHasANext(qs QuadIndexer, subIt iterator.Scanner, d quad.Direction) *hasANext { return &hasANext{ qs: qs, primary: subIt, dir: d, } } // Direction accessor. func (it *hasANext) Direction() quad.Direction { return it.dir } // Pass the TagResults down the chain. func (it *hasANext) TagResults(dst map[string]refs.Ref) { it.primary.TagResults(dst) } func (it *hasANext) String() string { return fmt.Sprintf("HasANext(%v)", it.dir) } // Get the next result that matches this branch. func (it *hasANext) NextPath(ctx context.Context) bool { return it.primary.NextPath(ctx) } // Next advances the iterator. This is simpler than Contains. We have a // subiterator we can get a value from, and we can take that resultant quad, // pull our direction out of it, and return that. func (it *hasANext) Next(ctx context.Context) bool { if !it.primary.Next(ctx) { return false } var err error it.result, err = it.qs.QuadDirection(it.primary.Result(), it.dir) if err != nil { it.err = err return false } return true } func (it *hasANext) Err() error { if it.err != nil { return it.err } return it.primary.Err() } func (it *hasANext) Result() refs.Ref { return it.result } // Close the subiterator, the result iterator (if any) and the HasA. It closes // all subiterators it can, but returns the first error it encounters. func (it *hasANext) Close() error { return it.primary.Close() } // A HasA consists of a reference back to the graph.QuadStore that it references, // a primary subiterator, a direction in which the quads for that subiterator point, // and a temporary holder for the iterator generated on Contains(). type hasAContains struct { qs QuadIndexer primary iterator.Index dir quad.Direction results iterator.Scanner result refs.Ref err error } // Construct a new HasA iterator, given the quad subiterator, and the quad // direction for which it stands. func newHasAContains(qs QuadIndexer, subIt iterator.Index, d quad.Direction) iterator.Index { return &hasAContains{ qs: qs, primary: subIt, dir: d, } } // Direction accessor. func (it *hasAContains) Direction() quad.Direction { return it.dir } // Pass the TagResults down the chain. func (it *hasAContains) TagResults(dst map[string]refs.Ref) { it.primary.TagResults(dst) } func (it *hasAContains) String() string { return fmt.Sprintf("HasAContains(%v)", it.dir) } // Check a value against our internal iterator. In order to do this, we must first open a new // iterator of "quads that have `val` in our direction", given to us by the quad store, // and then Next() values out of that iterator and Contains() them against our subiterator. func (it *hasAContains) Contains(ctx context.Context, val refs.Ref) bool { if clog.V(4) { clog.Infof("Id is %v", val) } // TODO(barakmich): Optimize this if it.results != nil { it.results.Close() } it.results = it.qs.QuadIterator(it.dir, val).Iterate() ok := it.nextContains(ctx) if it.err != nil { return false } return ok } // nextContains() is shared code between Contains() and GetNextResult() -- calls next on the // result iterator (a quad iterator based on the last checked value) and returns true if // another match is made. func (it *hasAContains) nextContains(ctx context.Context) bool { if it.results == nil { return false } for it.results.Next(ctx) { link := it.results.Result() if clog.V(4) { qlv, err := it.qs.Quad(link) if err == nil { clog.Infof("Quad is %v", qlv) } else { clog.Warningf("Error looking up result quad: %v", err) } } if it.primary.Contains(ctx, link) { var err error it.result, err = it.qs.QuadDirection(link, it.dir) if err != nil { it.err = err return false } return true } } it.err = it.results.Err() return false } // Get the next result that matches this branch. func (it *hasAContains) NextPath(ctx context.Context) bool { // Order here is important. If the subiterator has a NextPath, then we // need do nothing -- there is a next result, and we shouldn't move forward. // However, we then need to get the next result from our last Contains(). // // The upshot is, the end of NextPath() bubbles up from the bottom of the // iterator tree up, and we need to respect that. if clog.V(4) { clog.Infof("HASA %p NextPath", it) } if it.primary.NextPath(ctx) { return true } it.err = it.primary.Err() if it.err != nil { return false } result := it.nextContains(ctx) // Sets it.err if there's an error if it.err != nil { return false } if clog.V(4) { clog.Infof("HASA %p NextPath Returns %v", it, result) } return result } func (it *hasAContains) Err() error { return it.err } func (it *hasAContains) Result() refs.Ref { return it.result } // Close the subiterator, the result iterator (if any) and the HasA. It closes // all subiterators it can, but returns the first error it encounters. func (it *hasAContains) Close() error { err := it.primary.Close() if it.results != nil { if err2 := it.results.Close(); err2 != nil && err == nil { err = err2 } } return err } ================================================ FILE: graph/hasa_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph_test import ( "context" "errors" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/quad" ) func TestHasAIteratorErr(t *testing.T) { wantErr := errors.New("unique") ctx := context.TODO() errIt := iterator.NewError(wantErr) // TODO(andrew-d): pass a non-nil quadstore hasa := graph.NewHasA(nil, errIt, quad.Subject).Iterate() require.False(t, hasa.Next(ctx), "HasA iterator did not pass through initial 'false'") require.Equal(t, wantErr, hasa.Err(), "HasA iterator did not pass through underlying Err") } ================================================ FILE: graph/http/httpgraph.go ================================================ package httpgraph import ( "net/http" "github.com/cayleygraph/cayley/graph" ) type QuadStore interface { graph.QuadStore ForRequest(r *http.Request) (graph.QuadStore, error) } ================================================ FILE: graph/iterator/and.go ================================================ // Defines the And iterator, one of the base iterators. And requires no // knowledge of the constituent QuadStore; its sole purpose is to act as an // intersection operator across the subiterators it is given. If one iterator // contains [1,3,5] and another [2,3,4] -- then And is an iterator that // 'contains' [3] // // It accomplishes this in one of two ways. If it is a Next()ed iterator (that // is, it is a top level iterator, or on the "Next() path", then it will Next() // it's primary iterator (helpfully, and.primary_it) and Contains() the resultant // value against it's other iterators. If it matches all of them, then it // returns that value. Otherwise, it repeats the process. // // If it's on a Contains() path, it merely Contains()s every iterator, and returns the // logical AND of each result. package iterator import ( "context" "github.com/cayleygraph/cayley/graph/refs" ) // The And iterator. Consists of a number of subiterators, the primary of which will // be Next()ed if next is called. type And struct { sub []Shape checkList []Shape // special order for Contains opt []Shape } // NewAnd creates an And iterator. `qs` is only required when needing a handle // for QuadStore-specific optimizations, otherwise nil is acceptable. func NewAnd(sub ...Shape) *And { it := &And{ sub: make([]Shape, 0, 20), } for _, s := range sub { it.AddSubIterator(s) } return it } func (it *And) Iterate() Scanner { if len(it.sub) == 0 { return NewNull().Iterate() } sub := make([]Index, 0, len(it.sub)-1) for _, s := range it.sub[1:] { sub = append(sub, s.Lookup()) } opt := make([]Index, 0, len(it.opt)) for _, s := range it.opt { opt = append(opt, s.Lookup()) } return newAndNext(it.sub[0].Iterate(), newAndContains(sub, opt)) } func (it *And) Lookup() Index { if len(it.sub) == 0 { return NewNull().Lookup() } sub := make([]Index, 0, len(it.sub)) check := it.checkList if check == nil { check = it.sub } for _, s := range check { sub = append(sub, s.Lookup()) } opt := make([]Index, 0, len(it.opt)) for _, s := range it.opt { opt = append(opt, s.Lookup()) } return newAndContains(sub, opt) } // Returns a slice of the subiterators, in order (primary iterator first). func (it *And) SubIterators() []Shape { iters := make([]Shape, 0, len(it.sub)+len(it.opt)) iters = append(iters, it.sub...) iters = append(iters, it.opt...) return iters } func (it *And) String() string { return "And" } // Add a subiterator to this And iterator. // // The first iterator that is added becomes the primary iterator. This is // important. Calling Optimize() is the way to change the order based on // subiterator statistics. Without Optimize(), the order added is the order // used. func (it *And) AddSubIterator(sub Shape) { if sub == nil { panic("nil iterator") } it.sub = append(it.sub, sub) } // AddOptionalIterator adds an iterator that will only be Contain'ed and will not affect iteration results. // Only tags will be propagated from this iterator. func (it *And) AddOptionalIterator(sub Shape) *And { it.opt = append(it.opt, sub) return it } // The And iterator. Consists of a number of subiterators, the primary of which will // be Next()ed if next is called. type andNext struct { primary Scanner secondary Index result refs.Ref } // NewAnd creates an And iterator. `qs` is only required when needing a handle // for QuadStore-specific optimizations, otherwise nil is acceptable. func newAndNext(pri Scanner, sec Index) Scanner { return &andNext{ primary: pri, secondary: sec, } } // An extended TagResults, as it needs to add it's own results and // recurse down it's subiterators. func (it *andNext) TagResults(dst map[string]refs.Ref) { it.primary.TagResults(dst) it.secondary.TagResults(dst) } func (it *andNext) String() string { return "AndNext" } // Returns advances the And iterator. Because the And is the intersection of its // subiterators, it must choose one subiterator to produce a candidate, and check // this value against the subiterators. A productive choice of primary iterator // is therefore very important. func (it *andNext) Next(ctx context.Context) bool { for it.primary.Next(ctx) { cur := it.primary.Result() if it.secondary.Contains(ctx, cur) { it.result = cur return true } } return false } func (it *andNext) Err() error { if err := it.primary.Err(); err != nil { return err } if err := it.secondary.Err(); err != nil { return err } return nil } func (it *andNext) Result() refs.Ref { return it.result } // An And has no NextPath of its own -- that is, there are no other values // which satisfy our previous result that are not the result itself. Our // subiterators might, however, so just pass the call recursively. func (it *andNext) NextPath(ctx context.Context) bool { if it.primary.NextPath(ctx) { return true } else if err := it.primary.Err(); err != nil { return false } if it.secondary.NextPath(ctx) { return true } else if err := it.secondary.Err(); err != nil { return false } return false } // Close this iterator, and, by extension, close the subiterators. // Close should be idempotent, and it follows that if it's subiterators // follow this contract, the And follows the contract. It closes all // subiterators it can, but returns the first error it encounters. func (it *andNext) Close() error { err := it.primary.Close() if err2 := it.secondary.Close(); err2 != nil && err == nil { err = err2 } return err } // The And iterator. Consists of a number of subiterators, the primary of which will // be Next()ed if next is called. type andContains struct { base Shape sub []Index opt []Index optCheck []bool result refs.Ref err error } // NewAnd creates an And iterator. `qs` is only required when needing a handle // for QuadStore-specific optimizations, otherwise nil is acceptable. func newAndContains(sub, opt []Index) Index { return &andContains{ sub: sub, opt: opt, optCheck: make([]bool, len(opt)), } } // An extended TagResults, as it needs to add it's own results and // recurse down it's subiterators. func (it *andContains) TagResults(dst map[string]refs.Ref) { for _, sub := range it.sub { sub.TagResults(dst) } for i, sub := range it.opt { if !it.optCheck[i] { continue } sub.TagResults(dst) } } func (it *andContains) String() string { return "AndContains" } func (it *andContains) Err() error { if err := it.err; err != nil { return err } for _, si := range it.sub { if err := si.Err(); err != nil { return err } } for _, si := range it.opt { if err := si.Err(); err != nil { return err } } return nil } func (it *andContains) Result() refs.Ref { return it.result } // Check a value against the entire iterator, in order. func (it *andContains) Contains(ctx context.Context, val refs.Ref) bool { prev := it.result for i, sub := range it.sub { if !sub.Contains(ctx, val) { if err := sub.Err(); err != nil { it.err = err return false } // One of the iterators has determined that this value doesn't // match. However, the iterators that came before in the list // may have returned "ok" to Contains(). We need to set all // the tags back to what the previous result was -- effectively // seeking back exactly one -- so we check all the prior iterators // with the (already verified) result and throw away the result, // which will be 'true' if prev != nil { for j := 0; j < i; j++ { it.sub[j].Contains(ctx, prev) if err := it.sub[j].Err(); err != nil { it.err = err return false } } } return false } } it.result = val for i, sub := range it.opt { // remember if we will need to call TagResults on it, nothing more it.optCheck[i] = sub.Contains(ctx, val) } return true } // An And has no NextPath of its own -- that is, there are no other values // which satisfy our previous result that are not the result itself. Our // subiterators might, however, so just pass the call recursively. func (it *andContains) NextPath(ctx context.Context) bool { for _, sub := range it.sub { if sub.NextPath(ctx) { return true } else if err := sub.Err(); err != nil { it.err = err return false } } for i, sub := range it.opt { if !it.optCheck[i] { continue } if sub.NextPath(ctx) { return true } else if err := sub.Err(); err != nil { it.err = err return false } } return false } // Close this iterator, and, by extension, close the subiterators. // Close should be idempotent, and it follows that if it's subiterators // follow this contract, the And follows the contract. It closes all // subiterators it can, but returns the first error it encounters. func (it *andContains) Close() error { var err error for _, sub := range it.sub { if err2 := sub.Close(); err2 != nil && err == nil { err = err2 } } for _, sub := range it.opt { if err2 := sub.Close(); err2 != nil && err == nil { err = err2 } } return err } ================================================ FILE: graph/iterator/and_optimize.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator import ( "context" "sort" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph/refs" ) // Perhaps the most tricky file in this entire module. Really a method on the // And, but important enough to deserve its own file. // // Calling Optimize() on an And iterator, like any iterator, requires that we // preserve the underlying meaning. However, the And has many choices, namely, // which one of it's subiterators will be the branch that does the Next()ing, // and which ordering of the remaining iterators is the most efficient. In // short, this is where a lot of the query optimization happens, and there are // many wins to be had here, as well as many bad bugs. The worst class of bug // changes the meaning of the query. The second worst class makes things really // slow. // // The good news is this: If Optimize() is never called (turned off, perhaps) we can // be sure the results are as good as the query language called for. // // In short, tread lightly. // Optimizes the And, by picking the most efficient way to Next() and // Contains() its subiterators. For SQL fans, this is equivalent to JOIN. func (it *And) Optimize(ctx context.Context) (Shape, bool) { // First, let's get the slice of iterators, in order (first one is Next()ed, // the rest are Contains()ed) old := it.sub if len(old) == 0 { return NewNull(), true } // And call Optimize() on our subtree, replacing each one in the order we // found them. it_list is the newly optimized versions of these, and changed // is another list, of only the ones that have returned replacements and // changed. its := optimizeSubIterators(ctx, old) // If we can find only one subiterator which is equivalent to this whole and, // we can replace the And... if out := optimizeReplacement(its); out != nil && len(it.opt) == 0 { // ...And return it. return out, true } // And now, without changing any of the iterators, we reorder them. it_list is // now a permutation of itself, but the contents are unchanged. its = optimizeOrder(ctx, its) its, _ = materializeIts(ctx, its) // Okay! At this point we have an optimized order. // The easiest thing to do at this point is merely to create a new And iterator // and replace ourselves with our (reordered, optimized) clone. // Add the subiterators in order. newAnd := NewAnd(its...) opt := optimizeSubIterators(ctx, it.opt) for _, sub := range opt { newAnd.AddOptionalIterator(sub) } _ = newAnd.optimizeContains(ctx) if clog.V(3) { clog.Infof("%p become %p", it, newAnd) } return newAnd, true } // Find if there is a single subiterator which is a valid replacement for this // And. func optimizeReplacement(its []Shape) Shape { // If we were created with no SubIterators, we're as good as Null. if len(its) == 0 { return NewNull() } if len(its) == 1 { // When there's only one iterator, there's only one choice. return its[0] } // If any of our subiterators, post-optimization, are also Null, then // there's no point in continuing the branch, we will have no results // and we are null as well. if hasAnyNullIterators(its) { return NewNull() } return nil } // optimizeOrder(l) takes a list and returns a list, containing the same contents // but with a new ordering, however it wishes. func optimizeOrder(ctx context.Context, its []Shape) []Shape { var ( best Shape bestCost = int64(1 << 62) ) // Find the iterator with the projected "best" total cost. // Total cost is defined as The Next()ed iterator's cost to Next() out // all of it's contents, and to Contains() each of those against everyone // else. costs := make([]Costs, 0, len(its)) for _, it := range its { st, _ := it.Stats(ctx) costs = append(costs, st) } for i, root := range its { rootStats := costs[i] cost := rootStats.NextCost for j, f := range its { if f == root { continue } stats := costs[j] cost += stats.ContainsCost * (1 + (rootStats.Size.Value / (stats.Size.Value + 1))) } cost *= rootStats.Size.Value if clog.V(3) { clog.Infof("And: Root: %p Total Cost: %v Best: %v", root, cost, bestCost) } if cost < bestCost { best = root bestCost = cost } } if clog.V(3) { clog.Infof("And: Choosing: %p Best: %v", best, bestCost) } // TODO(barakmich): Optimization of order need not stop here. Picking a smart // Contains() order based on probability of getting a false Contains() first is // useful (fail faster). var out []Shape // Put the best iterator (the one we wish to Next()) at the front... if best != nil { out = append(out, best) } // ... push everyone else after... for _, it := range its { if it != best { out = append(out, it) } } return out } func sortByContainsCost(ctx context.Context, arr []Shape) error { cost := make([]Costs, 0, len(arr)) var last error for _, s := range arr { c, err := s.Stats(ctx) if err != nil { last = err } cost = append(cost, c) } sort.Sort(byCost{ list: arr, cost: cost, }) return last } // TODO(dennwc): store stats slice once type byCost struct { list []Shape cost []Costs } func (c byCost) Len() int { return len(c.list) } func (c byCost) Less(i, j int) bool { return c.cost[i].ContainsCost < c.cost[j].ContainsCost } func (c byCost) Swap(i, j int) { c.list[i], c.list[j] = c.list[j], c.list[i] c.cost[i], c.cost[j] = c.cost[j], c.cost[i] } // optimizeContains() creates an alternate check list, containing the same contents // but with a new ordering, however it wishes. func (it *And) optimizeContains(ctx context.Context) error { // GetSubIterators allocates, so this is currently safe. // TODO(kortschak) Reuse it.checkList if possible. // This involves providing GetSubIterators with a slice to fill. // Generally this is a worthwhile thing to do in other places as well. it.checkList = append([]Shape{}, it.sub...) return sortByContainsCost(ctx, it.checkList) } // optimizeSubIterators(l) takes a list of iterators and calls Optimize() on all // of them. It returns two lists -- the first contains the same list as l, where // any replacements are made by Optimize() and the second contains the originals // which were replaced. func optimizeSubIterators(ctx context.Context, its []Shape) []Shape { out := make([]Shape, 0, len(its)) for _, it := range its { o, _ := it.Optimize(ctx) out = append(out, o) } return out } // Check a list of iterators for any Null iterators. func hasAnyNullIterators(its []Shape) bool { for _, it := range its { if IsNull(it) { return true } } return false } func materializeIts(ctx context.Context, its []Shape) ([]Shape, error) { var out []Shape allStats, stats, err := getStatsForSlice(ctx, its, nil) out = append(out, its[0]) for i, it := range its[1:] { st := stats[i+1] if st.Size.Value*st.NextCost < (st.ContainsCost * (1 + (st.Size.Value / (allStats.Size.Value + 1)))) { if Height(it, func(it Shape) bool { _, ok := it.(*Materialize) return !ok }) > 10 { out = append(out, NewMaterialize(it)) continue } } out = append(out, it) } return out, err } func getStatsForSlice(ctx context.Context, its, opt []Shape) (Costs, []Costs, error) { if len(its) == 0 { return Costs{}, nil, nil } arr := make([]Costs, 0, len(its)) primaryStats, _ := its[0].Stats(ctx) arr = append(arr, primaryStats) containsCost := primaryStats.ContainsCost nextCost := primaryStats.NextCost size := primaryStats.Size.Value exact := primaryStats.Size.Exact var last error for _, sub := range its[1:] { stats, err := sub.Stats(ctx) if err != nil { last = err } arr = append(arr, stats) nextCost += stats.ContainsCost * (1 + (primaryStats.Size.Value / (stats.Size.Value + 1))) containsCost += stats.ContainsCost if size > stats.Size.Value { size = stats.Size.Value exact = stats.Size.Exact } } for _, sub := range opt { stats, _ := sub.Stats(ctx) nextCost += stats.ContainsCost * (1 + (primaryStats.Size.Value / (stats.Size.Value + 1))) containsCost += stats.ContainsCost } return Costs{ ContainsCost: containsCost, NextCost: nextCost, Size: refs.Size{ Value: size, Exact: exact, }, }, arr, last } // Stats lives here in and-iterator-optimize.go because it may // in the future return different statistics based on how it is optimized. // For now, however, it's pretty static. // // Returns the approximate size of the And iterator. Because we're dealing // with an intersection, we know that the largest we can be is the size of the // smallest iterator. This is the heuristic we shall follow. Better heuristics // welcome. func (it *And) Stats(ctx context.Context) (Costs, error) { stats, _, err := getStatsForSlice(ctx, it.sub, it.opt) return stats, err } ================================================ FILE: graph/iterator/and_optimize_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator_test // Tests relating to methods in and-iterator-optimize. Many are pretty simplistic, but // nonetheless cover a lot of basic cases. import ( "context" "testing" . "github.com/cayleygraph/cayley/graph/iterator" "github.com/stretchr/testify/require" ) func TestNullIteratorAnd(t *testing.T) { all := newInt64(1, 3, true) null := NewNull() a := NewAnd(all, null) newIt, changed := a.Optimize(context.TODO()) if !changed { t.Error("Didn't change") } if _, ok := newIt.(*Null); !ok { t.Errorf("Expected null iterator, got %T", newIt) } } func TestReorderWithTag(t *testing.T) { all := NewFixed(Int64Node(3)) all2 := NewFixed( Int64Node(3), Int64Node(4), Int64Node(5), Int64Node(6), ) a := NewAnd() // Make all2 the default iterator a.AddSubIterator(all2) a.AddSubIterator(all) _, changed := a.Optimize(context.TODO()) require.True(t, changed, "expected new iterator") } func TestAndStatistics(t *testing.T) { all := newInt64(100, 300, true) all2 := newInt64(1, 30000, true) a := NewAnd() // Make all2 the default iterator a.AddSubIterator(all2) a.AddSubIterator(all) ctx := context.TODO() stats1, _ := a.Stats(ctx) newIt, changed := a.Optimize(ctx) require.True(t, changed, "didn't optimize") stats2, _ := newIt.Stats(ctx) if stats2.NextCost > stats1.NextCost { t.Error("And didn't optimize. Next cost old ", stats1.NextCost, "and new ", stats2.NextCost) } } ================================================ FILE: graph/iterator/and_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator_test import ( "context" "errors" "testing" "github.com/stretchr/testify/require" . "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" ) // Make sure that tags work on the And. func TestAndTag(t *testing.T) { ctx := context.TODO() fix1 := NewFixed(Int64Node(234)) fix2 := NewFixed(Int64Node(234)) var ands Shape = NewAnd(Tag(fix1, "foo")).AddOptionalIterator(Tag(fix2, "baz")) ands = Tag(ands, "bar") and := ands.Iterate() require.True(t, and.Next(ctx)) require.Equal(t, Int64Node(234), and.Result()) tags := make(map[string]refs.Ref) and.TagResults(tags) require.Equal(t, map[string]refs.Ref{ "foo": Int64Node(234), "bar": Int64Node(234), "baz": Int64Node(234), }, tags) } // Do a simple itersection of fixed values. func TestAndAndFixedIterators(t *testing.T) { ctx := context.TODO() fix1 := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), Int64Node(4), ) fix2 := NewFixed( Int64Node(3), Int64Node(4), Int64Node(5), ) ands := NewAnd(fix1, fix2) // Should be as big as smallest subiterator st, err := ands.Stats(ctx) require.NoError(t, err) require.Equal(t, refs.Size{ Value: 3, Exact: true, }, st.Size) and := ands.Iterate() require.True(t, and.Next(ctx)) require.Equal(t, Int64Node(3), and.Result()) require.True(t, and.Next(ctx)) require.Equal(t, Int64Node(4), and.Result()) require.False(t, and.Next(ctx)) } // If there's no intersection, the size should still report the same, // but there should be nothing to Next() func TestNonOverlappingFixedIterators(t *testing.T) { ctx := context.TODO() fix1 := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), Int64Node(4), ) fix2 := NewFixed( Int64Node(5), Int64Node(6), Int64Node(7), ) ands := NewAnd(fix1, fix2) // Should be as big as smallest subiterator st, err := ands.Stats(ctx) require.NoError(t, err) require.Equal(t, refs.Size{ Value: 3, Exact: true, }, st.Size) and := ands.Iterate() require.False(t, and.Next(ctx)) } func TestAllIterators(t *testing.T) { ctx := context.TODO() all1 := newInt64(1, 5, true) all2 := newInt64(4, 10, true) and := NewAnd(all2, all1).Iterate() require.True(t, and.Next(ctx)) require.Equal(t, Int64Node(4), and.Result()) require.True(t, and.Next(ctx)) require.Equal(t, Int64Node(5), and.Result()) require.False(t, and.Next(ctx)) } func TestAndIteratorErr(t *testing.T) { ctx := context.TODO() wantErr := errors.New("unique") allErr := newTestIterator(false, wantErr) and := NewAnd( allErr, newInt64(1, 5, true), ).Iterate() require.False(t, and.Next(ctx)) require.Equal(t, wantErr, and.Err()) } ================================================ FILE: graph/iterator/count.go ================================================ package iterator import ( "context" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) // Count iterator returns one element with size of underlying iterator. type Count struct { it Shape qs refs.Namer } // NewCount creates a new iterator to count a number of results from a provided subiterator. // qs may be nil - it's used to check if count Contains (is) a given value. func NewCount(it Shape, qs refs.Namer) *Count { return &Count{ it: it, qs: qs, } } func (it *Count) Iterate() Scanner { return newCountNext(it.it) } func (it *Count) Lookup() Index { return newCountContains(it.it, it.qs) } // SubIterators returns a slice of the sub iterators. func (it *Count) SubIterators() []Shape { return []Shape{it.it} } func (it *Count) Optimize(ctx context.Context) (Shape, bool) { sub, optimized := it.it.Optimize(ctx) it.it = sub return it, optimized } func (it *Count) Stats(ctx context.Context) (Costs, error) { stats := Costs{ NextCost: 1, Size: refs.Size{ Value: 1, Exact: true, }, } if sub, err := it.it.Stats(ctx); err == nil && !sub.Size.Exact { stats.NextCost = sub.NextCost * sub.Size.Value } stats.ContainsCost = stats.NextCost return stats, nil } func (it *Count) String() string { return "Count" } // Count iterator returns one element with size of underlying iterator. type countNext struct { it Shape done bool result quad.Value err error } // NewCount creates a new iterator to count a number of results from a provided subiterator. // qs may be nil - it's used to check if count Contains (is) a given value. func newCountNext(it Shape) *countNext { return &countNext{ it: it, } } func (it *countNext) TagResults(dst map[string]refs.Ref) {} // Next counts a number of results in underlying iterator. func (it *countNext) Next(ctx context.Context) bool { if it.done { return false } // TODO(dennwc): this most likely won't include the NextPath st, err := it.it.Stats(ctx) if err != nil { it.err = err return false } if !st.Size.Exact { sit := it.it.Iterate() defer sit.Close() for st.Size.Value = 0; sit.Next(ctx); st.Size.Value++ { // TODO(dennwc): it's unclear if we should call it here or not for ; sit.NextPath(ctx); st.Size.Value++ { } } it.err = sit.Err() } it.result = quad.Int(st.Size.Value) it.done = true return true } func (it *countNext) Err() error { return it.err } func (it *countNext) Result() refs.Ref { if it.result == nil { return nil } return refs.PreFetched(it.result) } func (it *countNext) NextPath(ctx context.Context) bool { return false } func (it *countNext) Close() error { return nil } func (it *countNext) String() string { return "CountNext" } // Count iterator returns one element with size of underlying iterator. type countContains struct { it *countNext qs refs.Namer err error } // NewCount creates a new iterator to count a number of results from a provided subiterator. // qs may be nil - it's used to check if count Contains (is) a given value. func newCountContains(it Shape, qs refs.Namer) *countContains { return &countContains{ it: newCountNext(it), qs: qs, } } func (it *countContains) TagResults(dst map[string]refs.Ref) {} func (it *countContains) Err() error { if it.err != nil { return it.err } return it.it.Err() } func (it *countContains) Result() refs.Ref { return it.it.Result() } func (it *countContains) Contains(ctx context.Context, val refs.Ref) bool { if !it.it.done { it.it.Next(ctx) } if v, ok := val.(refs.PreFetchedValue); ok { return v.NameOf() == it.it.result } if it.qs != nil { valName, err := it.qs.NameOf(val) if err != nil { it.err = err return false } return valName == it.it.result } return false } func (it *countContains) NextPath(ctx context.Context) bool { return false } func (it *countContains) Close() error { return it.it.Close() } func (it *countContains) String() string { return "CountContains" } ================================================ FILE: graph/iterator/count_test.go ================================================ package iterator import ( "context" "testing" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" ) func TestCount(t *testing.T) { ctx := context.TODO() fixed := NewFixed( refs.PreFetched(quad.String("a")), refs.PreFetched(quad.String("b")), refs.PreFetched(quad.String("c")), refs.PreFetched(quad.String("d")), refs.PreFetched(quad.String("e")), ) its := NewCount(fixed, nil) itn := its.Iterate() require.True(t, itn.Next(ctx)) require.Equal(t, refs.PreFetched(quad.Int(5)), itn.Result()) require.False(t, itn.Next(ctx)) itc := its.Lookup() require.True(t, itc.Contains(ctx, refs.PreFetched(quad.Int(5)))) require.False(t, itc.Contains(ctx, refs.PreFetched(quad.Int(3)))) fixed2 := NewFixed( refs.PreFetched(quad.String("b")), refs.PreFetched(quad.String("d")), ) its = NewCount(NewAnd(fixed, fixed2), nil) itn = its.Iterate() require.True(t, itn.Next(ctx)) require.Equal(t, refs.PreFetched(quad.Int(2)), itn.Result()) require.False(t, itn.Next(ctx)) itc = its.Lookup() require.False(t, itc.Contains(ctx, refs.PreFetched(quad.Int(5)))) require.True(t, itc.Contains(ctx, refs.PreFetched(quad.Int(2)))) } ================================================ FILE: graph/iterator/fixed.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator // Defines one of the base iterators, the Fixed iterator. A fixed iterator is quite simple; it // contains an explicit fixed array of values. // // A fixed iterator requires an Equality function to be passed to it, by reason that refs.Ref, the // opaque Quad store value, may not answer to ==. import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" ) var _ Shape = &Fixed{} // A Fixed iterator consists of it's values, an index (where it is in the process of Next()ing) and // an equality function. type Fixed struct { values []refs.Ref } // NewFixed creates a new Fixed iterator with a custom comparator. func NewFixed(vals ...refs.Ref) *Fixed { return &Fixed{ values: append([]refs.Ref{}, vals...), } } func (it *Fixed) Iterate() Scanner { return newFixedNext(it.values) } func (it *Fixed) Lookup() Index { return newFixedContains(it.values) } // Add a value to the iterator. The array now contains this value. // TODO(barakmich): This ought to be a set someday, disallowing repeated values. func (it *Fixed) Add(v refs.Ref) { it.values = append(it.values, v) } // Values returns a list of values stored in iterator. Slice must not be modified. func (it *Fixed) Values() []refs.Ref { return it.values } func (it *Fixed) String() string { return fmt.Sprintf("Fixed(%v)", it.values) } // No sub-iterators. func (it *Fixed) SubIterators() []Shape { return nil } // Optimize() for a Fixed iterator is simple. Returns a Null iterator if it's empty // (so that other iterators upstream can treat this as null) or there is no // optimization. func (it *Fixed) Optimize(ctx context.Context) (Shape, bool) { if len(it.values) == 1 && it.values[0] == nil { return NewNull(), true } return it, false } // As we right now have to scan the entire list, Next and Contains are linear with the // size. However, a better data structure could remove these limits. func (it *Fixed) Stats(ctx context.Context) (Costs, error) { return Costs{ ContainsCost: 1, NextCost: 1, Size: refs.Size{ Value: int64(len(it.values)), Exact: true, }, }, nil } // A Fixed iterator consists of it's values, an index (where it is in the process of Next()ing) and // an equality function. type fixedNext struct { values []refs.Ref ind int result refs.Ref } // Creates a new Fixed iterator with a custom comparator. func newFixedNext(vals []refs.Ref) *fixedNext { return &fixedNext{ values: vals, } } func (it *fixedNext) Close() error { return nil } func (it *fixedNext) TagResults(dst map[string]refs.Ref) {} func (it *fixedNext) String() string { return fmt.Sprintf("Fixed(%v)", it.values) } // Next advances the iterator. func (it *fixedNext) Next(ctx context.Context) bool { if it.ind >= len(it.values) { return false } out := it.values[it.ind] it.result = out it.ind++ return true } func (it *fixedNext) Err() error { return nil } func (it *fixedNext) Result() refs.Ref { return it.result } func (it *fixedNext) NextPath(ctx context.Context) bool { return false } // A Fixed iterator consists of it's values, an index (where it is in the process of Next()ing) and // an equality function. type fixedContains struct { values []refs.Ref keys []interface{} result refs.Ref } // Creates a new Fixed iterator with a custom comparator. func newFixedContains(vals []refs.Ref) *fixedContains { keys := make([]interface{}, 0, len(vals)) for _, v := range vals { keys = append(keys, refs.ToKey(v)) } return &fixedContains{ values: vals, keys: keys, } } func (it *fixedContains) Close() error { return nil } func (it *fixedContains) TagResults(dst map[string]refs.Ref) {} func (it *fixedContains) String() string { return fmt.Sprintf("Fixed(%v)", it.values) } // Check if the passed value is equal to one of the values stored in the iterator. func (it *fixedContains) Contains(ctx context.Context, v refs.Ref) bool { // Could be optimized by keeping it sorted or using a better datastructure. // However, for fixed iterators, which are by definition kind of tiny, this // isn't a big issue. vk := refs.ToKey(v) for i, x := range it.keys { if x == vk { it.result = it.values[i] return true } } return false } func (it *fixedContains) Err() error { return nil } func (it *fixedContains) Result() refs.Ref { return it.result } func (it *fixedContains) NextPath(ctx context.Context) bool { return false } ================================================ FILE: graph/iterator/iterate.go ================================================ package iterator import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) // Chain is a chain-enabled helper to setup iterator execution. type Chain struct { ctx context.Context s Shape it Scanner qs refs.Namer paths bool optimize bool limit int n int } // Iterate is a set of helpers for iteration. Context may be used to cancel execution. // Iterator will be optimized and closed after execution. // // By default, iteration has no limit and includes sub-paths. func Iterate(ctx context.Context, it Shape) *Chain { if ctx == nil { ctx = context.Background() } return &Chain{ ctx: ctx, s: it, limit: -1, paths: true, optimize: true, } } func (c *Chain) next() bool { select { case <-c.ctx.Done(): return false default: } ok := (c.limit < 0 || c.n < c.limit) && c.it.Next(c.ctx) if ok { c.n++ } return ok } func (c *Chain) nextPath() bool { select { case <-c.ctx.Done(): return false default: } ok := c.paths && (c.limit < 0 || c.n < c.limit) && c.it.NextPath(c.ctx) if ok { c.n++ } return ok } func (c *Chain) start() { if c.optimize { c.s, _ = c.s.Optimize(c.ctx) } c.it = c.s.Iterate() } func (c *Chain) end() { c.it.Close() } // Limit limits a total number of results returned. func (c *Chain) Limit(n int) *Chain { c.limit = n return c } // Paths switches iteration over sub-paths (with it.NextPath). // Defaults to true. func (c *Chain) Paths(enable bool) *Chain { c.paths = enable return c } // On sets a default quad store for iteration. If qs was set, it may be omitted in other functions. func (c *Chain) On(qs refs.Namer) *Chain { c.qs = qs return c } // UnOptimized disables iterator optimization. func (c *Chain) UnOptimized() *Chain { c.optimize = false return c } // Each will run a provided callback for each result of the iterator. func (c *Chain) Each(fnc func(refs.Ref) error) error { c.start() defer c.end() done := c.ctx.Done() for c.next() { select { case <-done: return c.ctx.Err() default: } err := fnc(c.it.Result()) if err != nil { return err } for c.nextPath() { select { case <-done: return c.ctx.Err() default: } fnc(c.it.Result()) } } return c.it.Err() } // All will return all results of an iterator. func (c *Chain) Count() (int64, error) { // TODO(dennwc): this should wrap the shape in Count if c.optimize { c.s, _ = c.s.Optimize(c.ctx) } if st, err := c.s.Stats(c.ctx); err != nil { return st.Size.Value, err } else if st.Size.Exact { return st.Size.Value, nil } c.start() defer c.end() if err := c.it.Err(); err != nil { return 0, err } done := c.ctx.Done() var cnt int64 iteration: for c.next() { select { case <-done: break iteration default: } cnt++ for c.nextPath() { select { case <-done: break iteration default: } cnt++ } } return cnt, c.it.Err() } // All will return all results of an iterator. func (c *Chain) All() ([]refs.Ref, error) { c.start() defer c.end() done := c.ctx.Done() var out []refs.Ref iteration: for c.next() { select { case <-done: break iteration default: } out = append(out, c.it.Result()) for c.nextPath() { select { case <-done: break iteration default: } out = append(out, c.it.Result()) } } return out, c.it.Err() } // First will return a first result of an iterator. It returns nil if iterator is empty. func (c *Chain) First() (refs.Ref, error) { c.start() defer c.end() if !c.next() { return nil, c.it.Err() } return c.it.Result(), nil } // Send will send each result of the iterator to the provided channel. // // Channel will NOT be closed when function returns. func (c *Chain) Send(out chan<- refs.Ref) error { c.start() defer c.end() done := c.ctx.Done() for c.next() { select { case <-done: return c.ctx.Err() case out <- c.it.Result(): } for c.nextPath() { select { case <-done: return c.ctx.Err() case out <- c.it.Result(): } } } return c.it.Err() } // TagEach will run a provided tag map callback for each result of the iterator. func (c *Chain) TagEach(fnc func(map[string]refs.Ref) error) error { c.start() defer c.end() done := c.ctx.Done() mn := 0 for c.next() { select { case <-done: return c.ctx.Err() default: } tags := make(map[string]refs.Ref, mn) c.it.TagResults(tags) if n := len(tags); n > mn { mn = n } err := fnc(tags) if err != nil { return err } for c.nextPath() { select { case <-done: return c.ctx.Err() default: } tags := make(map[string]refs.Ref, mn) c.it.TagResults(tags) if n := len(tags); n > mn { mn = n } err = fnc(tags) if err != nil { return err } } } return c.it.Err() } var errNoQuadStore = fmt.Errorf("no quad store in Iterate") // EachValue is an analog of Each, but it will additionally call NameOf // for each graph.Ref before passing it to a callback. func (c *Chain) EachValue(qs refs.Namer, fnc func(quad.Value) error) error { if qs != nil { c.qs = qs } if c.qs == nil { return errNoQuadStore } // TODO(dennwc): batch NameOf? return c.Each(func(v refs.Ref) error { nv, err := c.qs.NameOf(v) if err == nil && nv != nil { err = fnc(nv) } return err }) } // EachValuePair is an analog of Each, but it will additionally call NameOf // for each graph.Ref before passing it to a callback. Original value will be passed as well. func (c *Chain) EachValuePair(qs refs.Namer, fnc func(refs.Ref, quad.Value) error) error { if qs != nil { c.qs = qs } if c.qs == nil { return errNoQuadStore } // TODO(dennwc): batch NameOf? return c.Each(func(v refs.Ref) error { nv, err := c.qs.NameOf(v) if err == nil && nv != nil { err = fnc(v, nv) } return err }) } // AllValues is an analog of All, but it will additionally call NameOf // for each graph.Ref before returning the results slice. func (c *Chain) AllValues(qs refs.Namer) ([]quad.Value, error) { var out []quad.Value err := c.EachValue(qs, func(v quad.Value) error { out = append(out, v) return nil }) return out, err } // FirstValue is an analog of First, but it does lookup of a value in QuadStore. func (c *Chain) FirstValue(qs refs.Namer) (quad.Value, error) { if qs != nil { c.qs = qs } if c.qs == nil { return nil, errNoQuadStore } v, err := c.First() if err != nil || v == nil { return nil, err } return c.qs.NameOf(v) } // SendValues is an analog of Send, but it will additionally call NameOf // for each graph.Ref before sending it to a channel. func (c *Chain) SendValues(qs refs.Namer, out chan<- quad.Value) error { if qs != nil { c.qs = qs } if c.qs == nil { return errNoQuadStore } c.start() defer c.end() done := c.ctx.Done() send := func(v refs.Ref) error { nv, err := c.qs.NameOf(c.it.Result()) if err != nil || nv == nil { return err } nvResult, err := c.qs.NameOf(c.it.Result()) if err != nil { return err } select { case <-done: return c.ctx.Err() case out <- nvResult: } return nil } for c.next() { if err := send(c.it.Result()); err != nil { return err } for c.nextPath() { if err := send(c.it.Result()); err != nil { return err } } } return c.it.Err() } // TagValues is an analog of TagEach, but it will additionally call NameOf // for each graph.Ref before passing the map to a callback. func (c *Chain) TagValues(qs refs.Namer, fnc func(map[string]quad.Value) error) error { if qs != nil { c.qs = qs } if c.qs == nil { return errNoQuadStore } return c.TagEach(func(m map[string]refs.Ref) error { vm := make(map[string]quad.Value, len(m)) for k, v := range m { var err error vm[k], err = c.qs.NameOf(v) // TODO(dennwc): batch NameOf? if err != nil { return err } } fnc(vm) return nil }) } ================================================ FILE: graph/iterator/iterator.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator // Define the general iterator interface. import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" ) var ( _ Shape = &Null{} _ Shape = &Error{} ) // TaggerBase is a base interface for Tagger and TaggerShape. type TaggerBase interface { Tags() []string FixedTags() map[string]refs.Ref AddTags(tag ...string) AddFixedTag(tag string, value refs.Ref) } // Base is a set of common methods for Scanner and Index iterators. type Base interface { // String returns a short textual representation of an iterator. String() string // Fills a tag-to-result-value map. TagResults(map[string]refs.Ref) // Returns the current result. Result() refs.Ref // These methods are the heart and soul of the iterator, as they constitute // the iteration interface. // // To get the full results of iteration, do the following: // // for it.Next(ctx) { // val := it.Result() // ... do things with val. // for it.NextPath(ctx) { // ... find other paths to iterate // } // } // // All of them should set iterator.result to be the last returned value, to // make results work. // // NextPath() advances iterators that may have more than one valid result, // from the bottom up. NextPath(ctx context.Context) bool // Err returns any error that was encountered by the Iterator. Err() error // TODO: make a requirement that Err should return ErrClosed after Close is called // Close the iterator and do internal cleanup. Close() error } // Scanner is an iterator that lists all results sequentially, but not necessarily in a sorted order. type Scanner interface { Base // Next advances the iterator to the next value, which will then be available through // the Result method. It returns false if no further advancement is possible, or if an // error was encountered during iteration. Err should be consulted to distinguish // between the two cases. Next(ctx context.Context) bool } // Index is an index lookup iterator. It allows to check if an index contains a specific value. type Index interface { Base // Contains returns whether the value is within the set held by the iterator. // // It will set Result to the matching subtree. TagResults can be used to collect values from tree branches. Contains(ctx context.Context, v refs.Ref) bool } // TaggerShape is an interface for iterators that can tag values. Tags are returned as a part of TagResults call. type TaggerShape interface { Shape TaggerBase CopyFromTagger(st TaggerBase) } type Costs struct { ContainsCost int64 NextCost int64 Size refs.Size } // Shape is an iterator shape, similar to a query plan. But the plan is not specific in this // case - it is used to reorder query branches, and the decide what branches will be scanned // and what branches will lookup values (hopefully from the index, but not necessarily). type Shape interface { // TODO(dennwc): merge with shape.Shape // String returns a short textual representation of an iterator. String() string // Iterate starts this iterator in scanning mode. Resulting iterator will list all // results sequentially, but not necessary in the sorted order. Caller must close // the iterator. Iterate() Scanner // Lookup starts this iterator in an index lookup mode. Depending on the iterator type, // this may still involve database scans. Resulting iterator allows to check an index // contains a specified value. Caller must close the iterator. Lookup() Index // These methods relate to choosing the right iterator, or optimizing an // iterator tree // // Stats() returns the relative costs of calling the iteration methods for // this iterator, as well as the size. Roughly, it will take NextCost * Size // "cost units" to get everything out of the iterator. This is a wibbly-wobbly // thing, and not exact, but a useful heuristic. Stats(ctx context.Context) (Costs, error) // Optimizes an iterator. Can replace the iterator, or merely move things // around internally. if it chooses to replace it with a better iterator, // returns (the new iterator, true), if not, it returns (self, false). Optimize(ctx context.Context) (Shape, bool) // Return a slice of the subiterators for this iterator. SubIterators() []Shape } type Morphism func(Shape) Shape func IsNull(it Shape) bool { if _, ok := it.(*Null); ok { return true } return false } // Height is a convienence function to measure the height of an iterator tree. func Height(it Shape, filter func(Shape) bool) int { if filter != nil && !filter(it) { return 1 } subs := it.SubIterators() maxDepth := 0 for _, sub := range subs { h := Height(sub, filter) if h > maxDepth { maxDepth = h } } return maxDepth + 1 } // Null is the simplest iterator -- the Null iterator. It contains nothing. // It is the empty set. Often times, queries that contain one of these match nothing, // so it's important to give it a special iterator. type Null struct{} // NewNull creates a new Null iterator // Fairly useless New function. func NewNull() *Null { return &Null{} } // Iterate implements Iterator func (it *Null) Iterate() Scanner { return it } // Lookup implements Iterator func (it *Null) Lookup() Index { return it } // Fill the map based on the tags assigned to this iterator. func (it *Null) TagResults(dst map[string]refs.Ref) {} func (it *Null) Contains(ctx context.Context, v refs.Ref) bool { return false } // A good iterator will close itself when it returns true. // Null has nothing it needs to do. func (it *Null) Optimize(ctx context.Context) (Shape, bool) { return it, false } func (it *Null) String() string { return "Null" } func (it *Null) Next(ctx context.Context) bool { return false } func (it *Null) Err() error { return nil } func (it *Null) Result() refs.Ref { return nil } func (it *Null) SubIterators() []Shape { return nil } func (it *Null) NextPath(ctx context.Context) bool { return false } func (it *Null) Reset() {} func (it *Null) Close() error { return nil } // A null iterator costs nothing. Use it! func (it *Null) Stats(ctx context.Context) (Costs, error) { return Costs{}, nil } // Error iterator always returns a single error with no other results. type Error struct { err error } func NewError(err error) *Error { return &Error{err: err} } func (it *Error) Iterate() Scanner { return it } func (it *Error) Lookup() Index { return it } // Fill the map based on the tags assigned to this iterator. func (it *Error) TagResults(dst map[string]refs.Ref) {} func (it *Error) Contains(ctx context.Context, v refs.Ref) bool { return false } func (it *Error) Optimize(ctx context.Context) (Shape, bool) { return it, false } func (it *Error) String() string { return fmt.Sprintf("Error(%v)", it.err) } func (it *Error) Next(ctx context.Context) bool { return false } func (it *Error) Err() error { return it.err } func (it *Error) Result() refs.Ref { return nil } func (it *Error) SubIterators() []Shape { return nil } func (it *Error) NextPath(ctx context.Context) bool { return false } func (it *Error) Reset() {} func (it *Error) Close() error { return it.err } func (it *Error) Stats(ctx context.Context) (Costs, error) { return Costs{}, nil } ================================================ FILE: graph/iterator/iterator_test.go ================================================ package iterator_test import ( "context" "fmt" . "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" ) // A testing iterator that returns the given values for Next() and Err(). type testIterator struct { Shape NextVal bool ErrVal error } func newTestIterator(next bool, err error) Shape { return &testIterator{ Shape: NewFixed(), NextVal: next, ErrVal: err, } } func (it *testIterator) Iterate() Scanner { return &testIteratorNext{ Scanner: it.Shape.Iterate(), NextVal: it.NextVal, ErrVal: it.ErrVal, } } func (it *testIterator) Lookup() Index { return &testIteratorContains{ Index: it.Shape.Lookup(), NextVal: it.NextVal, ErrVal: it.ErrVal, } } // A testing iterator that returns the given values for Next() and Err(). type testIteratorNext struct { Scanner NextVal bool ErrVal error } func (it *testIteratorNext) Next(ctx context.Context) bool { return it.NextVal } func (it *testIteratorNext) Err() error { return it.ErrVal } // A testing iterator that returns the given values for Next() and Err(). type testIteratorContains struct { Index NextVal bool ErrVal error } func (it *testIteratorContains) Contains(ctx context.Context, v refs.Ref) bool { return it.NextVal } func (it *testIteratorContains) Err() error { return it.ErrVal } type Int64Quad int64 func (v Int64Quad) Key() interface{} { return v } func (Int64Quad) IsNode() bool { return false } var _ Shape = &Int64{} // An All iterator across a range of int64 values, from `max` to `min`. type Int64 struct { node bool max, min int64 } func (it *Int64) Iterate() Scanner { return newInt64Next(it.min, it.max, it.node) } func (it *Int64) Lookup() Index { return newInt64Contains(it.min, it.max, it.node) } // Creates a new Int64 with the given range. func newInt64(min, max int64, node bool) *Int64 { return &Int64{ node: node, min: min, max: max, } } func (it *Int64) String() string { return fmt.Sprintf("Int64(%d-%d)", it.min, it.max) } // No sub-iterators. func (it *Int64) SubIterators() []Shape { return nil } // The number of elements in an Int64 is the size of the range. // The size is exact. func (it *Int64) Size() (int64, bool) { sz := (it.max - it.min) + 1 return sz, true } func valToInt64(v refs.Ref) int64 { if v, ok := v.(Int64Node); ok { return int64(v) } return int64(v.(Int64Quad)) } // There's nothing to optimize about this little iterator. func (it *Int64) Optimize(ctx context.Context) (Shape, bool) { return it, false } // Stats for an Int64 are simple. Super cheap to do any operation, // and as big as the range. func (it *Int64) Stats(ctx context.Context) (Costs, error) { s, exact := it.Size() return Costs{ ContainsCost: 1, NextCost: 1, Size: refs.Size{ Value: s, Exact: exact, }, }, nil } // An All iterator across a range of int64 values, from `max` to `min`. type int64Next struct { node bool max, min int64 at int64 result int64 } // Creates a new Int64 with the given range. func newInt64Next(min, max int64, node bool) *int64Next { return &int64Next{ node: node, min: min, max: max, at: min, } } func (it *int64Next) Close() error { return nil } func (it *int64Next) TagResults(dst map[string]refs.Ref) {} func (it *int64Next) String() string { return fmt.Sprintf("Int64(%d-%d)", it.min, it.max) } // Next() on an Int64 all iterator is a simple incrementing counter. // Return the next integer, and mark it as the result. func (it *int64Next) Next(ctx context.Context) bool { if it.at == -1 { return false } val := it.at it.at = it.at + 1 if it.at > it.max { it.at = -1 } it.result = val return true } func (it *int64Next) Err() error { return nil } func (it *int64Next) toValue(v int64) refs.Ref { if it.node { return Int64Node(v) } return Int64Quad(v) } func (it *int64Next) Result() refs.Ref { return it.toValue(it.result) } func (it *int64Next) NextPath(ctx context.Context) bool { return false } // An All iterator across a range of int64 values, from `max` to `min`. type int64Contains struct { node bool max, min int64 at int64 result int64 } // Creates a new Int64 with the given range. func newInt64Contains(min, max int64, node bool) *int64Contains { return &int64Contains{ node: node, min: min, max: max, at: min, } } func (it *int64Contains) Close() error { return nil } func (it *int64Contains) TagResults(dst map[string]refs.Ref) {} func (it *int64Contains) String() string { return fmt.Sprintf("Int64(%d-%d)", it.min, it.max) } func (it *int64Contains) Err() error { return nil } func (it *int64Contains) toValue(v int64) refs.Ref { if it.node { return Int64Node(v) } return Int64Quad(v) } func (it *int64Contains) Result() refs.Ref { return it.toValue(it.result) } func (it *int64Contains) NextPath(ctx context.Context) bool { return false } // No sub-iterators. func (it *int64Contains) SubIterators() []Shape { return nil } // Contains() for an Int64 is merely seeing if the passed value is // within the range, assuming the value is an int64. func (it *int64Contains) Contains(ctx context.Context, tsv refs.Ref) bool { v := valToInt64(tsv) if it.min <= v && v <= it.max { it.result = v return true } return false } ================================================ FILE: graph/iterator/limit.go ================================================ package iterator import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" ) // Limit iterator will stop iterating if certain a number of values were encountered. // Zero and negative Limit values means no Limit. type Limit struct { limit int64 it Shape } func NewLimit(it Shape, max int64) *Limit { return &Limit{ limit: max, it: it, } } func (it *Limit) Iterate() Scanner { return NewLimitNext(it.it.Iterate(), it.limit) } func (it *Limit) Lookup() Index { return newLimitContains(it.it.Lookup(), it.limit) } // SubIterators returns a slice of the sub iterators. func (it *Limit) SubIterators() []Shape { return []Shape{it.it} } func (it *Limit) Optimize(ctx context.Context) (Shape, bool) { nit, optimized := it.it.Optimize(ctx) if it.limit <= 0 { // no Limit return nit, true } it.it = nit return it, optimized } func (it *Limit) Stats(ctx context.Context) (Costs, error) { st, err := it.it.Stats(ctx) if it.limit > 0 && st.Size.Value > it.limit { st.Size.Value = it.limit } return st, err } func (it *Limit) String() string { return fmt.Sprintf("Limit(%d)", it.limit) } // Limit iterator will stop iterating if certain a number of values were encountered. // Zero and negative Limit values means no Limit. type limitNext struct { limit int64 count int64 it Scanner } func NewLimitNext(it Scanner, limit int64) Scanner { return &limitNext{ limit: limit, it: it, } } func (it *limitNext) TagResults(dst map[string]refs.Ref) { it.it.TagResults(dst) } // Next advances the Limit iterator. It will stop iteration if Limit was reached. func (it *limitNext) Next(ctx context.Context) bool { if it.limit > 0 && it.count >= it.limit { return false } if it.it.Next(ctx) { it.count++ return true } return false } func (it *limitNext) Err() error { return it.it.Err() } func (it *limitNext) Result() refs.Ref { return it.it.Result() } // NextPath checks whether there is another path. Will call primary iterator // if Limit is not reached yet. func (it *limitNext) NextPath(ctx context.Context) bool { if it.limit > 0 && it.count >= it.limit { return false } if it.it.NextPath(ctx) { it.count++ return true } return false } // Close closes the primary and all iterators. It closes all subiterators // it can, but returns the first error it encounters. func (it *limitNext) Close() error { return it.it.Close() } func (it *limitNext) String() string { return fmt.Sprintf("LimitNext(%d)", it.limit) } // Limit iterator will stop iterating if certain a number of values were encountered. // Zero and negative Limit values means no Limit. type limitContains struct { limit int64 count int64 it Index } func newLimitContains(it Index, limit int64) *limitContains { return &limitContains{ limit: limit, it: it, } } func (it *limitContains) TagResults(dst map[string]refs.Ref) { it.it.TagResults(dst) } func (it *limitContains) Err() error { return it.it.Err() } func (it *limitContains) Result() refs.Ref { return it.it.Result() } func (it *limitContains) Contains(ctx context.Context, val refs.Ref) bool { if it.limit > 0 && it.count >= it.limit { return false } if it.it.Contains(ctx, val) { it.count++ return true } return false } // NextPath checks whether there is another path. Will call primary iterator // if Limit is not reached yet. func (it *limitContains) NextPath(ctx context.Context) bool { if it.limit > 0 && it.count >= it.limit { return false } if it.it.NextPath(ctx) { it.count++ return true } return false } // Close closes the primary and all iterators. It closes all subiterators // it can, but returns the first error it encounters. func (it *limitContains) Close() error { return it.it.Close() } func (it *limitContains) String() string { return fmt.Sprintf("LimitContains(%d)", it.limit) } ================================================ FILE: graph/iterator/limit_test.go ================================================ package iterator_test import ( "context" "testing" "github.com/stretchr/testify/require" . "github.com/cayleygraph/cayley/graph/iterator" ) func TestLimitIteratorBasics(t *testing.T) { ctx := context.TODO() allIt := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), Int64Node(4), Int64Node(5), ) u := NewLimit(allIt, 0) expectSz, _ := allIt.Stats(ctx) sz, _ := u.Stats(ctx) require.Equal(t, expectSz.Size.Value, sz.Size.Value) require.Equal(t, []int{1, 2, 3, 4, 5}, iterated(u)) u = NewLimit(allIt, 3) sz, _ = u.Stats(ctx) require.Equal(t, int64(3), sz.Size.Value) require.Equal(t, []int{1, 2, 3}, iterated(u)) uc := u.Lookup() for _, v := range []int{1, 2, 3} { require.True(t, uc.Contains(ctx, Int64Node(v))) } require.False(t, uc.Contains(ctx, Int64Node(4))) uc = u.Lookup() for _, v := range []int{5, 4, 3} { require.True(t, uc.Contains(ctx, Int64Node(v))) } require.False(t, uc.Contains(ctx, Int64Node(2))) } ================================================ FILE: graph/iterator/materialize.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator // A simple iterator that, when first called Contains() or Next() upon, materializes the whole subiterator, stores it locally, and responds. Essentially a cache. import ( "context" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph/refs" ) const MaterializeLimit = 1000 type result struct { id refs.Ref tags map[string]refs.Ref } type Materialize struct { sub Shape expectSize int64 } func NewMaterialize(sub Shape) *Materialize { return NewMaterializeWithSize(sub, 0) } func NewMaterializeWithSize(sub Shape, size int64) *Materialize { return &Materialize{ sub: sub, expectSize: size, } } func (it *Materialize) Iterate() Scanner { return newMaterializeNext(it.sub) } func (it *Materialize) Lookup() Index { return newMaterializeContains(it.sub) } func (it *Materialize) String() string { return "Materialize" } func (it *Materialize) SubIterators() []Shape { return []Shape{it.sub} } func (it *Materialize) Optimize(ctx context.Context) (Shape, bool) { newSub, changed := it.sub.Optimize(ctx) if changed { it.sub = newSub if IsNull(it.sub) { return it.sub, true } } return it, false } // The entire point of Materialize is to amortize the cost by // putting it all up front. func (it *Materialize) Stats(ctx context.Context) (Costs, error) { overhead := int64(2) var size refs.Size subitStats, err := it.sub.Stats(ctx) if it.expectSize > 0 { size = refs.Size{Value: it.expectSize, Exact: false} } else { size = subitStats.Size } return Costs{ ContainsCost: overhead * subitStats.NextCost, NextCost: overhead * subitStats.NextCost, Size: size, }, err } type materializeNext struct { sub Shape next Scanner containsMap map[interface{}]int values [][]result index int subindex int hasRun bool aborted bool err error } func newMaterializeNext(sub Shape) *materializeNext { return &materializeNext{ containsMap: make(map[interface{}]int), sub: sub, next: sub.Iterate(), index: -1, } } func (it *materializeNext) Close() error { it.containsMap = nil it.values = nil it.hasRun = false return it.next.Close() } func (it *materializeNext) TagResults(dst map[string]refs.Ref) { if !it.hasRun { return } if it.aborted { it.next.TagResults(dst) return } if it.Result() == nil { return } for tag, value := range it.values[it.index][it.subindex].tags { dst[tag] = value } } func (it *materializeNext) String() string { return "Materialize" } func (it *materializeNext) Result() refs.Ref { if it.aborted { return it.next.Result() } if len(it.values) == 0 { return nil } if it.index == -1 { return nil } if it.index >= len(it.values) { return nil } return it.values[it.index][it.subindex].id } func (it *materializeNext) Next(ctx context.Context) bool { if !it.hasRun { it.materializeSet(ctx) } if it.err != nil { return false } if it.aborted { n := it.next.Next(ctx) it.err = it.next.Err() return n } it.index++ it.subindex = 0 if it.index >= len(it.values) { return false } return true } func (it *materializeNext) Err() error { return it.err } func (it *materializeNext) NextPath(ctx context.Context) bool { if !it.hasRun { it.materializeSet(ctx) } if it.err != nil { return false } if it.aborted { return it.next.NextPath(ctx) } it.subindex++ if it.subindex >= len(it.values[it.index]) { // Don't go off the end of the world it.subindex-- return false } return true } func (it *materializeNext) materializeSet(ctx context.Context) { i := 0 mn := 0 for it.next.Next(ctx) { i++ if i > MaterializeLimit { it.aborted = true break } id := it.next.Result() val := refs.ToKey(id) if _, ok := it.containsMap[val]; !ok { it.containsMap[val] = len(it.values) it.values = append(it.values, nil) } index := it.containsMap[val] tags := make(map[string]refs.Ref, mn) it.next.TagResults(tags) if n := len(tags); n > mn { mn = n } it.values[index] = append(it.values[index], result{id: id, tags: tags}) for it.next.NextPath(ctx) { i++ if i > MaterializeLimit { it.aborted = true break } tags := make(map[string]refs.Ref, mn) it.next.TagResults(tags) if n := len(tags); n > mn { mn = n } it.values[index] = append(it.values[index], result{id: id, tags: tags}) } } it.err = it.next.Err() if it.err == nil && it.aborted { if clog.V(2) { clog.Infof("Aborting subiterator") } it.values = nil it.containsMap = nil _ = it.next.Close() it.next = it.sub.Iterate() } it.hasRun = true } type materializeContains struct { next *materializeNext sub Index // only set if aborted } func newMaterializeContains(sub Shape) *materializeContains { return &materializeContains{ next: newMaterializeNext(sub), } } func (it *materializeContains) Close() error { err := it.next.Close() if it.sub != nil { if err2 := it.sub.Close(); err2 != nil && err == nil { err = err2 } } return err } func (it *materializeContains) TagResults(dst map[string]refs.Ref) { if it.sub != nil { it.sub.TagResults(dst) return } it.next.TagResults(dst) } func (it *materializeContains) String() string { return "MaterializeContains" } func (it *materializeContains) Result() refs.Ref { if it.sub != nil { return it.sub.Result() } return it.next.Result() } func (it *materializeContains) Err() error { if err := it.next.Err(); err != nil { return err } else if it.sub == nil { return nil } return it.sub.Err() } func (it *materializeContains) run(ctx context.Context) { it.next.materializeSet(ctx) if it.next.aborted { it.sub = it.next.sub.Lookup() } } func (it *materializeContains) Contains(ctx context.Context, v refs.Ref) bool { if !it.next.hasRun { it.run(ctx) } if it.next.Err() != nil { return false } if it.sub != nil { return it.sub.Contains(ctx, v) } key := refs.ToKey(v) if i, ok := it.next.containsMap[key]; ok { it.next.index = i it.next.subindex = 0 return true } return false } func (it *materializeContains) NextPath(ctx context.Context) bool { if !it.next.hasRun { it.run(ctx) } if it.next.Err() != nil { return false } if it.sub != nil { return it.sub.NextPath(ctx) } return it.next.NextPath(ctx) } ================================================ FILE: graph/iterator/materialize_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator_test import ( "context" "errors" "testing" "github.com/stretchr/testify/require" . "github.com/cayleygraph/cayley/graph/iterator" ) func TestMaterializeIteratorError(t *testing.T) { ctx := context.TODO() wantErr := errors.New("unique") errIt := newTestIterator(false, wantErr) // This tests that we properly return 0 results and the error when the // underlying iterator returns an error. mIt := NewMaterialize(errIt).Iterate() require.False(t, mIt.Next(ctx)) require.Equal(t, wantErr, mIt.Err()) } func TestMaterializeIteratorErrorAbort(t *testing.T) { ctx := context.TODO() wantErr := errors.New("unique") errIt := newTestIterator(false, wantErr) // This tests that we properly return 0 results and the error when the // underlying iterator is larger than our 'abort at' value, and then // returns an error. or := NewOr( newInt64(1, int64(MaterializeLimit+1), true), errIt, ) mIt := NewMaterialize(or).Iterate() // We should get all the underlying values... for i := 0; i < MaterializeLimit+1; i++ { require.True(t, mIt.Next(ctx)) require.NoError(t, mIt.Err()) } // ... and then the error value. require.False(t, mIt.Next(ctx)) require.Equal(t, wantErr, mIt.Err()) } ================================================ FILE: graph/iterator/misc.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator // Defines one of the base iterators, the All iterator. Which, logically // enough, represents all nodes or all links in the graph. // // This particular file is actually vestigial. It's up to the QuadStore to give // us an All iterator that represents all things in the graph. So this is // really the All iterator for the memstore.QuadStore. That said, it *is* one of // the base iterators, and it helps just to see it here. type Int64Node int64 func (v Int64Node) Key() interface{} { return v } func (Int64Node) IsNode() bool { return true } ================================================ FILE: graph/iterator/not.go ================================================ package iterator import ( "context" "github.com/cayleygraph/cayley/graph/refs" ) // Not iterator acts like a complement for the primary iterator. // It will return all the vertices which are not part of the primary iterator. type Not struct { primary Shape allIt Shape } func NewNot(primaryIt, allIt Shape) *Not { return &Not{ primary: primaryIt, allIt: allIt, } } func (it *Not) Iterate() Scanner { return newNotNext(it.primary.Lookup(), it.allIt.Iterate()) } func (it *Not) Lookup() Index { return newNotContains(it.primary.Lookup()) } // SubIterators returns a slice of the sub iterators. // The first iterator is the primary iterator, for which the complement // is generated. func (it *Not) SubIterators() []Shape { return []Shape{it.primary, it.allIt} } func (it *Not) Optimize(ctx context.Context) (Shape, bool) { // TODO - consider wrapping the primary with a MaterializeIt optimizedPrimaryIt, optimized := it.primary.Optimize(ctx) if optimized { it.primary = optimizedPrimaryIt } it.primary = NewMaterialize(it.primary) return it, false } func (it *Not) Stats(ctx context.Context) (Costs, error) { primaryStats, err := it.primary.Stats(ctx) allStats, err2 := it.allIt.Stats(ctx) if err == nil { err = err2 } return Costs{ NextCost: allStats.NextCost + primaryStats.ContainsCost, ContainsCost: primaryStats.ContainsCost, Size: refs.Size{ Value: allStats.Size.Value - primaryStats.Size.Value, Exact: false, }, }, err } func (it *Not) String() string { return "Not" } // Not iterator acts like a complement for the primary iterator. // It will return all the vertices which are not part of the primary iterator. type notNext struct { primaryIt Index allIt Scanner result refs.Ref } func newNotNext(primaryIt Index, allIt Scanner) *notNext { return ¬Next{ primaryIt: primaryIt, allIt: allIt, } } func (it *notNext) TagResults(dst map[string]refs.Ref) { if it.primaryIt != nil { it.primaryIt.TagResults(dst) } } // Next advances the Not iterator. It returns whether there is another valid // new value. It fetches the next value of the all iterator which is not // contained by the primary iterator. func (it *notNext) Next(ctx context.Context) bool { for it.allIt.Next(ctx) { if curr := it.allIt.Result(); !it.primaryIt.Contains(ctx, curr) { it.result = curr return true } } return false } func (it *notNext) Err() error { if err := it.allIt.Err(); err != nil { return err } if err := it.primaryIt.Err(); err != nil { return err } return nil } func (it *notNext) Result() refs.Ref { return it.result } // NextPath checks whether there is another path. Not applicable, hence it will // return false. func (it *notNext) NextPath(ctx context.Context) bool { return false } // Close closes the primary and all iterators. It closes all subiterators // it can, but returns the first error it encounters. func (it *notNext) Close() error { err := it.primaryIt.Close() if err2 := it.allIt.Close(); err2 != nil && err == nil { err = err2 } return err } func (it *notNext) String() string { return "NotNext" } // Not iterator acts like a complement for the primary iterator. // It will return all the vertices which are not part of the primary iterator. type notContains struct { primaryIt Index result refs.Ref err error } func newNotContains(primaryIt Index) *notContains { return ¬Contains{ primaryIt: primaryIt, } } func (it *notContains) TagResults(dst map[string]refs.Ref) { if it.primaryIt != nil { it.primaryIt.TagResults(dst) } } func (it *notContains) Err() error { return it.err } func (it *notContains) Result() refs.Ref { return it.result } // Contains checks whether the passed value is part of the primary iterator's // complement. For a valid value, it updates the Result returned by the iterator // to the value itself. func (it *notContains) Contains(ctx context.Context, val refs.Ref) bool { if it.primaryIt.Contains(ctx, val) { return false } it.err = it.primaryIt.Err() if it.err != nil { // Explicitly return 'false', since an error occurred. return false } it.result = val return true } // NextPath checks whether there is another path. Not applicable, hence it will // return false. func (it *notContains) NextPath(ctx context.Context) bool { return false } // Close closes the primary and all iterators. It closes all subiterators // it can, but returns the first error it encounters. func (it *notContains) Close() error { return it.primaryIt.Close() } func (it *notContains) String() string { return "NotContains" } ================================================ FILE: graph/iterator/not_test.go ================================================ package iterator_test import ( "context" "errors" "testing" "github.com/stretchr/testify/require" . "github.com/cayleygraph/cayley/graph/iterator" ) func TestNotIteratorBasics(t *testing.T) { ctx := context.TODO() allIt := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), Int64Node(4), ) toComplementIt := NewFixed( Int64Node(2), Int64Node(4), ) not := NewNot(toComplementIt, allIt) st, _ := not.Stats(ctx) require.Equal(t, int64(2), st.Size.Value) expect := []int{1, 3} for i := 0; i < 2; i++ { require.Equal(t, expect, iterated(not)) } nc := not.Lookup() for _, v := range []int{1, 3} { require.True(t, nc.Contains(ctx, Int64Node(v))) } for _, v := range []int{2, 4} { require.False(t, nc.Contains(ctx, Int64Node(v))) } } func TestNotIteratorErr(t *testing.T) { ctx := context.TODO() wantErr := errors.New("unique") allIt := newTestIterator(false, wantErr) toComplementIt := NewFixed() not := NewNot(toComplementIt, allIt).Iterate() require.False(t, not.Next(ctx)) require.Equal(t, wantErr, not.Err()) } ================================================ FILE: graph/iterator/or.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator // Defines the or and short-circuiting or iterator. Or is the union operator for it's subiterators. // Short-circuiting-or is a little different. It will return values from the first iterator that returns // values at all, and then stops. // // Never reorders the iterators from the order they arrive. It is either the union or the first one. // May return the same value twice -- once for each branch. import ( "context" "github.com/cayleygraph/cayley/graph/refs" ) type Or struct { isShortCircuiting bool sub []Shape curInd int result refs.Ref err error } func NewOr(sub ...Shape) *Or { it := &Or{ sub: make([]Shape, 0, 20), curInd: -1, } for _, s := range sub { it.AddSubIterator(s) } return it } func NewShortCircuitOr(sub ...Shape) *Or { it := &Or{ sub: make([]Shape, 0, 20), isShortCircuiting: true, curInd: -1, } for _, s := range sub { it.AddSubIterator(s) } return it } func (it *Or) Iterate() Scanner { sub := make([]Scanner, 0, len(it.sub)) for _, s := range it.sub { sub = append(sub, s.Iterate()) } return newOrNext(sub, it.isShortCircuiting) } func (it *Or) Lookup() Index { sub := make([]Index, 0, len(it.sub)) for _, s := range it.sub { sub = append(sub, s.Lookup()) } return newOrContains(sub, it.isShortCircuiting) } // Returns a list.List of the subiterators, in order. The returned slice must not be modified. func (it *Or) SubIterators() []Shape { return it.sub } func (it *Or) String() string { return "Or" } // Add a subiterator to this Or iterator. Order matters. func (it *Or) AddSubIterator(sub Shape) { it.sub = append(it.sub, sub) } func (it *Or) Optimize(ctx context.Context) (Shape, bool) { old := it.SubIterators() optIts := optimizeSubIterators(ctx, old) newOr := NewOr() newOr.isShortCircuiting = it.isShortCircuiting // Add the subiterators in order. for _, o := range optIts { newOr.AddSubIterator(o) } return newOr, true } // Returns the approximate size of the Or iterator. Because we're dealing // with a union, we know that the largest we can be is the sum of all the iterators, // or in the case of short-circuiting, the longest. func (it *Or) Stats(ctx context.Context) (Costs, error) { ContainsCost := int64(0) NextCost := int64(0) Size := refs.Size{ Value: 0, Exact: true, } var last error for _, sub := range it.sub { stats, err := sub.Stats(ctx) if err != nil { last = err } NextCost += stats.NextCost ContainsCost += stats.ContainsCost if it.isShortCircuiting { if Size.Value < stats.Size.Value { Size = stats.Size } } else { Size.Value += stats.Size.Value Size.Exact = Size.Exact && stats.Size.Exact } } return Costs{ ContainsCost: ContainsCost, NextCost: NextCost, Size: Size, }, last } type orNext struct { shortCircuit bool sub []Scanner curInd int result refs.Ref err error } func newOrNext(sub []Scanner, shortCircuit bool) *orNext { return &orNext{ sub: sub, curInd: -1, shortCircuit: shortCircuit, } } // Overrides BaseIterator TagResults, as it needs to add it's own results and // recurse down it's subiterators. func (it *orNext) TagResults(dst map[string]refs.Ref) { it.sub[it.curInd].TagResults(dst) } func (it *orNext) String() string { return "OrNext" } // Next advances the Or iterator. Because the Or is the union of its // subiterators, it must produce from all subiterators -- unless it it // shortcircuiting, in which case, it is the first one that returns anything. func (it *orNext) Next(ctx context.Context) bool { if it.curInd >= len(it.sub) { return false } var first bool for { if it.curInd == -1 { it.curInd = 0 first = true } curIt := it.sub[it.curInd] if curIt.Next(ctx) { it.result = curIt.Result() return true } it.err = curIt.Err() if it.err != nil { return false } if it.shortCircuit && !first { break } it.curInd++ if it.curInd >= len(it.sub) { break } } return false } func (it *orNext) Err() error { return it.err } func (it *orNext) Result() refs.Ref { return it.result } // An Or has no NextPath of its own -- that is, there are no other values // which satisfy our previous result that are not the result itself. Our // subiterators might, however, so just pass the call recursively. In the case of // shortcircuiting, only allow new results from the currently checked iterator func (it *orNext) NextPath(ctx context.Context) bool { if it.curInd != -1 { currIt := it.sub[it.curInd] ok := currIt.NextPath(ctx) if !ok { it.err = currIt.Err() } return ok } return false } // Close this iterator, and, by extension, close the subiterators. // Close should be idempotent, and it follows that if it's subiterators // follow this contract, the Or follows the contract. It closes all // subiterators it can, but returns the first error it encounters. func (it *orNext) Close() error { var err error for _, sub := range it.sub { _err := sub.Close() if _err != nil && err == nil { err = _err } } return err } type orContains struct { shortCircuit bool sub []Index curInd int result refs.Ref err error } func newOrContains(sub []Index, shortCircuit bool) *orContains { return &orContains{ sub: sub, curInd: -1, shortCircuit: shortCircuit, } } // Overrides BaseIterator TagResults, as it needs to add it's own results and // recurse down it's subiterators. func (it *orContains) TagResults(dst map[string]refs.Ref) { it.sub[it.curInd].TagResults(dst) } func (it *orContains) String() string { return "OrContains" } func (it *orContains) Err() error { return it.err } func (it *orContains) Result() refs.Ref { return it.result } // Checks a value against the iterators, in order. func (it *orContains) subItsContain(ctx context.Context, val refs.Ref) (bool, error) { subIsGood := false for i, sub := range it.sub { subIsGood = sub.Contains(ctx, val) if subIsGood { it.curInd = i break } err := sub.Err() if err != nil { return false, err } } return subIsGood, nil } // Check a value against the entire iterator, in order. func (it *orContains) Contains(ctx context.Context, val refs.Ref) bool { anyGood, err := it.subItsContain(ctx, val) if err != nil { it.err = err return false } else if !anyGood { return false } it.result = val return true } // An Or has no NextPath of its own -- that is, there are no other values // which satisfy our previous result that are not the result itself. Our // subiterators might, however, so just pass the call recursively. In the case of // shortcircuiting, only allow new results from the currently checked iterator func (it *orContains) NextPath(ctx context.Context) bool { if it.curInd != -1 { currIt := it.sub[it.curInd] ok := currIt.NextPath(ctx) if !ok { it.err = currIt.Err() } return ok } // TODO(dennwc): this should probably list matches from other sub-iterators return false } // Close this iterator, and, by extension, close the subiterators. // Close should be idempotent, and it follows that if it's subiterators // follow this contract, the Or follows the contract. It closes all // subiterators it can, but returns the first error it encounters. func (it *orContains) Close() error { var err error for _, sub := range it.sub { _err := sub.Close() if _err != nil && err == nil { err = _err } } return err } ================================================ FILE: graph/iterator/or_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator_test import ( "context" "errors" "testing" "github.com/stretchr/testify/require" . "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" ) func iterated(s Shape) []int { ctx := context.TODO() var res []int it := s.Iterate() defer it.Close() for it.Next(ctx) { res = append(res, int(it.Result().(Int64Node))) } return res } func TestOrIteratorBasics(t *testing.T) { ctx := context.TODO() or := NewOr() f1 := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), ) f2 := NewFixed( Int64Node(3), Int64Node(9), Int64Node(20), Int64Node(21), ) or.AddSubIterator(f1) or.AddSubIterator(f2) st, _ := or.Stats(ctx) require.Equal(t, int64(7), st.Size.Value) expect := []int{1, 2, 3, 3, 9, 20, 21} for i := 0; i < 2; i++ { require.Equal(t, expect, iterated(or)) } // Check that optimization works. optOr, _ := or.Optimize(ctx) require.Equal(t, expect, iterated(optOr)) orc := or.Lookup() for _, v := range []int{2, 3, 21} { require.True(t, orc.Contains(ctx, Int64Node(v))) } for _, v := range []int{22, 5, 0} { require.False(t, orc.Contains(ctx, Int64Node(v))) } } func TestShortCircuitingOrBasics(t *testing.T) { ctx := context.TODO() var or *Or f1 := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), ) f2 := NewFixed( Int64Node(3), Int64Node(9), Int64Node(20), Int64Node(21), ) or = NewShortCircuitOr() or.AddSubIterator(f1) or.AddSubIterator(f2) st, _ := or.Stats(ctx) require.Equal(t, refs.Size{ Value: 4, Exact: true, }, st.Size) // It should extract the first iterators' numbers. or = NewShortCircuitOr() or.AddSubIterator(f1) or.AddSubIterator(f2) expect := []int{1, 2, 3} for i := 0; i < 2; i++ { require.Equal(t, expect, iterated(or)) } // Check optimization works. optOr, _ := or.Optimize(ctx) require.Equal(t, expect, iterated(optOr)) // Check that numbers in either iterator exist. or = NewShortCircuitOr() or.AddSubIterator(f1) or.AddSubIterator(f2) orc := or.Lookup() for _, v := range []int{2, 3, 21} { require.True(t, orc.Contains(ctx, Int64Node(v))) } for _, v := range []int{22, 5, 0} { require.False(t, orc.Contains(ctx, Int64Node(v))) } // Check that it pulls the second iterator's numbers if the first is empty. or = NewShortCircuitOr() or.AddSubIterator(NewFixed()) or.AddSubIterator(f2) expect = []int{3, 9, 20, 21} for i := 0; i < 2; i++ { require.Equal(t, expect, iterated(or)) } // Check optimization works. optOr, _ = or.Optimize(ctx) require.Equal(t, expect, iterated(optOr)) } func TestOrIteratorErr(t *testing.T) { ctx := context.TODO() wantErr := errors.New("unique") orErr := newTestIterator(false, wantErr) fix1 := NewFixed(Int64Node(1)) or := NewOr( fix1, orErr, newInt64(1, 5, true), ).Iterate() require.True(t, or.Next(ctx)) require.Equal(t, Int64Node(1), or.Result()) require.False(t, or.Next(ctx)) require.Equal(t, wantErr, or.Err()) } func TestShortCircuitOrIteratorErr(t *testing.T) { ctx := context.TODO() wantErr := errors.New("unique") orErr := newTestIterator(false, wantErr) or := NewOr( orErr, newInt64(1, 5, true), ).Iterate() require.False(t, or.Next(ctx)) require.Equal(t, wantErr, or.Err()) } ================================================ FILE: graph/iterator/recursive.go ================================================ package iterator import ( "context" "math" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) const recursiveBaseTag = "__base_recursive" type seenAt struct { depth int tags map[string]refs.Ref val refs.Ref } var DefaultMaxRecursiveSteps = 50 // Recursive iterator takes a base iterator and a morphism to be applied recursively, for each result. type Recursive struct { subIt Shape morphism Morphism maxDepth int depthTags []string } func NewRecursive(it Shape, morphism Morphism, maxDepth int) *Recursive { if maxDepth == 0 { maxDepth = DefaultMaxRecursiveSteps } return &Recursive{ subIt: it, morphism: morphism, maxDepth: maxDepth, } } func (it *Recursive) Iterate() Scanner { return newRecursiveNext(it.subIt.Iterate(), it.morphism, it.maxDepth, it.depthTags) } func (it *Recursive) Lookup() Index { return newRecursiveContains(newRecursiveNext(it.subIt.Iterate(), it.morphism, it.maxDepth, it.depthTags)) } func (it *Recursive) AddDepthTag(s string) { it.depthTags = append(it.depthTags, s) } func (it *Recursive) SubIterators() []Shape { return []Shape{it.subIt} } func (it *Recursive) Optimize(ctx context.Context) (Shape, bool) { newIt, optimized := it.subIt.Optimize(ctx) if optimized { it.subIt = newIt } return it, false } func (it *Recursive) Stats(ctx context.Context) (Costs, error) { base := NewFixed() base.Add(Int64Node(20)) fanoutit := it.morphism(base) fanoutStats, err := fanoutit.Stats(ctx) subitStats, err2 := it.subIt.Stats(ctx) if err == nil { err = err2 } size := int64(math.Pow(float64(subitStats.Size.Value*fanoutStats.Size.Value), 5)) return Costs{ NextCost: subitStats.NextCost + fanoutStats.NextCost, ContainsCost: (subitStats.NextCost+fanoutStats.NextCost)*(size/10) + subitStats.ContainsCost, Size: refs.Size{ Value: size, Exact: false, }, }, err } func (it *Recursive) String() string { return "Recursive" } // Recursive iterator takes a base iterator and a morphism to be applied recursively, for each result. type recursiveNext struct { subIt Scanner result seenAt err error morphism Morphism seen map[interface{}]seenAt nextIt Scanner depth int maxDepth int pathMap map[interface{}][]map[string]refs.Ref pathIndex int containsValue refs.Ref depthTags []string depthCache []refs.Ref baseIt *Fixed } func newRecursiveNext(it Scanner, morphism Morphism, maxDepth int, depthTags []string) *recursiveNext { return &recursiveNext{ subIt: it, morphism: morphism, maxDepth: maxDepth, depthTags: depthTags, seen: make(map[interface{}]seenAt), nextIt: &Null{}, baseIt: NewFixed(), pathMap: make(map[interface{}][]map[string]refs.Ref), } } func (it *recursiveNext) TagResults(dst map[string]refs.Ref) { for _, tag := range it.depthTags { dst[tag] = refs.PreFetched(quad.Int(it.result.depth)) } if it.containsValue != nil { paths := it.pathMap[refs.ToKey(it.containsValue)] if len(paths) != 0 { for k, v := range paths[it.pathIndex] { dst[k] = v } } } if it.nextIt != nil { it.nextIt.TagResults(dst) delete(dst, recursiveBaseTag) } } func (it *recursiveNext) Next(ctx context.Context) bool { it.pathIndex = 0 if it.depth == 0 { for it.subIt.Next(ctx) { res := it.subIt.Result() it.depthCache = append(it.depthCache, it.subIt.Result()) tags := make(map[string]refs.Ref) it.subIt.TagResults(tags) key := refs.ToKey(res) it.pathMap[key] = append(it.pathMap[key], tags) for it.subIt.NextPath(ctx) { tags := make(map[string]refs.Ref) it.subIt.TagResults(tags) it.pathMap[key] = append(it.pathMap[key], tags) } } } for { if !it.nextIt.Next(ctx) { if it.maxDepth > 0 && it.depth >= it.maxDepth { return false } else if len(it.depthCache) == 0 { return false } it.depth++ it.baseIt = NewFixed(it.depthCache...) it.depthCache = nil if it.nextIt != nil { it.nextIt.Close() } it.nextIt = it.morphism(Tag(it.baseIt, recursiveBaseTag)).Iterate() continue } val := it.nextIt.Result() results := make(map[string]refs.Ref) it.nextIt.TagResults(results) key := refs.ToKey(val) if _, seen := it.seen[key]; !seen { base := results[recursiveBaseTag] delete(results, recursiveBaseTag) it.seen[key] = seenAt{ val: base, depth: it.depth, tags: results, } it.result.depth = it.depth it.result.val = val it.containsValue = it.getBaseValue(val) it.depthCache = append(it.depthCache, val) return true } } } func (it *recursiveNext) Err() error { return it.err } func (it *recursiveNext) Result() refs.Ref { return it.result.val } func (it *recursiveNext) getBaseValue(val refs.Ref) refs.Ref { var at seenAt var ok bool if at, ok = it.seen[refs.ToKey(val)]; !ok { panic("trying to getBaseValue of something unseen") } for at.depth != 1 { if at.depth == 0 { panic("seen chain is broken") } at = it.seen[refs.ToKey(at.val)] } return at.val } func (it *recursiveNext) NextPath(ctx context.Context) bool { if it.pathIndex+1 >= len(it.pathMap[refs.ToKey(it.containsValue)]) { return false } it.pathIndex++ return true } func (it *recursiveNext) Close() error { err := it.subIt.Close() if err != nil { return err } err = it.nextIt.Close() if err != nil { return err } it.seen = nil return it.err } func (it *recursiveNext) String() string { return "RecursiveNext" } // Recursive iterator takes a base iterator and a morphism to be applied recursively, for each result. type recursiveContains struct { next *recursiveNext tags map[string]refs.Ref } func newRecursiveContains(next *recursiveNext) *recursiveContains { return &recursiveContains{ next: next, } } func (it *recursiveContains) TagResults(dst map[string]refs.Ref) { it.next.TagResults(dst) for k, v := range it.tags { dst[k] = v } } func (it *recursiveContains) Err() error { return it.next.Err() } func (it *recursiveContains) Result() refs.Ref { return it.next.Result() } func (it *recursiveContains) Contains(ctx context.Context, val refs.Ref) bool { it.next.pathIndex = 0 key := refs.ToKey(val) if at, ok := it.next.seen[key]; ok { it.next.containsValue = it.next.getBaseValue(val) it.next.result.depth = at.depth it.next.result.val = val it.tags = at.tags return true } for it.next.Next(ctx) { if refs.ToKey(it.next.Result()) == key { return true } } return false } func (it *recursiveContains) NextPath(ctx context.Context) bool { return it.next.NextPath(ctx) } func (it *recursiveContains) Close() error { return it.next.Close() } func (it *recursiveContains) String() string { return "RecursiveContains(" + it.next.String() + ")" } ================================================ FILE: graph/iterator/recursive_test.go ================================================ // Copyright 2015 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator_test import ( "context" "sort" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphmock" . "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) func singleHop(qs graph.QuadIndexer, pred string) Morphism { return func(it Shape) Shape { fixed := NewFixed() fixed.Add(refs.PreFetched(quad.Raw(pred))) predlto := graph.NewLinksTo(qs, fixed, quad.Predicate) lto := graph.NewLinksTo(qs, it, quad.Subject) and := NewAnd() and.AddSubIterator(lto) and.AddSubIterator(predlto) return graph.NewHasA(qs, and, quad.Object) } } var recTestQs = &graphmock.Store{ Data: []quad.Quad{ quad.MakeRaw("alice", "parent", "bob", ""), quad.MakeRaw("bob", "parent", "charlie", ""), quad.MakeRaw("charlie", "parent", "dani", ""), quad.MakeRaw("charlie", "parent", "bob", ""), quad.MakeRaw("dani", "parent", "emily", ""), quad.MakeRaw("fred", "follows", "alice", ""), quad.MakeRaw("greg", "follows", "alice", ""), }, } func TestRecursiveNext(t *testing.T) { ctx := context.TODO() qs := recTestQs start := NewFixed() start.Add(refs.PreFetched(quad.Raw("alice"))) r := NewRecursive(start, singleHop(qs, "parent"), 0).Iterate() expected := []string{"bob", "charlie", "dani", "emily"} var got []string for r.Next(ctx) { qn, err := qs.NameOf(r.Result()) require.NoError(t, err) got = append(got, quad.ToString(qn)) } sort.Strings(expected) sort.Strings(got) require.Equal(t, expected, got) } func TestRecursiveContains(t *testing.T) { ctx := context.TODO() qs := recTestQs start := NewFixed() start.Add(refs.PreFetched(quad.Raw("alice"))) r := NewRecursive(start, singleHop(qs, "parent"), 0).Lookup() values := []string{"charlie", "bob", "not"} expected := []bool{true, true, false} for i, v := range values { vn, err := qs.ValueOf(quad.Raw(v)) require.NoError(t, err) ok := r.Contains(ctx, vn) require.Equal(t, expected[i], ok) } } func TestRecursiveNextPath(t *testing.T) { ctx := context.TODO() qs := recTestQs start := qs.NodesAllIterator() start = Tag(start, "person") it := singleHop(qs, "follows")(start) and := NewAnd() and.AddSubIterator(it) fixed := NewFixed() fixed.Add(refs.PreFetched(quad.Raw("alice"))) and.AddSubIterator(fixed) r := NewRecursive(and, singleHop(qs, "parent"), 0).Iterate() expected := []string{"fred", "fred", "fred", "fred", "greg", "greg", "greg", "greg"} var got []string for r.Next(ctx) { res := make(map[string]refs.Ref) r.TagResults(res) vn, err := qs.NameOf(res["person"]) require.NoError(t, err) got = append(got, quad.ToString(vn)) for r.NextPath(ctx) { res := make(map[string]refs.Ref) r.TagResults(res) vn, err := qs.NameOf(res["person"]) require.NoError(t, err) got = append(got, quad.ToString(vn)) } } sort.Strings(expected) sort.Strings(got) require.Equal(t, expected, got) } ================================================ FILE: graph/iterator/regex.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator import ( "regexp" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) func newRegex(qs refs.Namer, sub Shape, re *regexp.Regexp, refs bool) Shape { return NewValueFilter(qs, sub, func(v quad.Value) (bool, error) { switch v := v.(type) { case quad.String: return re.MatchString(string(v)), nil case quad.LangString: return re.MatchString(string(v.Value)), nil case quad.TypedString: return re.MatchString(string(v.Value)), nil default: if refs { switch v := v.(type) { case quad.BNode: return re.MatchString(string(v)), nil case quad.IRI: return re.MatchString(string(v)), nil } } } return false, nil }) } // NewRegex returns an unary operator -- a filter across the values in the relevant // subiterator. It works similarly to gremlin's filter{it.matches('exp')}, // reducing the iterator set to values whose string representation passes a // regular expression test. func NewRegex(sub Shape, re *regexp.Regexp, qs refs.Namer) Shape { return newRegex(qs, sub, re, false) } // NewRegexWithRefs is like NewRegex but allows regexp iterator to match IRIs and BNodes. // // Consider using it carefully. In most cases it's better to reconsider // your graph structure instead of relying on slow unoptimizable regexp. // // An example of incorrect usage is to match IRIs: // // // Via regexp like: // http://example.org/page.* // // The right way is to explicitly link graph nodes and query them by this relation: // func NewRegexWithRefs(sub Shape, re *regexp.Regexp, qs refs.Namer) Shape { return newRegex(qs, sub, re, true) } ================================================ FILE: graph/iterator/resolver.go ================================================ // Copyright 2018 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) // A Resolver iterator consists of it's order, an index (where it is in the, // process of iterating) and a store to resolve values from. type Resolver struct { qs refs.Namer order []quad.Value } // NewResolver creates a new Resolver iterator. func NewResolver(qs refs.Namer, nodes ...quad.Value) *Resolver { it := &Resolver{ qs: qs, order: make([]quad.Value, len(nodes)), } copy(it.order, nodes) return it } func (it *Resolver) Iterate() Scanner { return newResolverNext(it.qs, it.order) } func (it *Resolver) Lookup() Index { return newResolverContains(it.qs, it.order) } func (it *Resolver) String() string { return fmt.Sprintf("Resolver(%v)", it.order) } func (it *Resolver) SubIterators() []Shape { return nil } // Returns a Null iterator if it's empty so that upstream iterators can optimize it // away, otherwise there is no optimization. func (it *Resolver) Optimize(ctx context.Context) (Shape, bool) { if len(it.order) == 0 { return NewNull(), true } return it, false } func (it *Resolver) Stats(ctx context.Context) (Costs, error) { return Costs{ // Next is (presumably) O(1) from store NextCost: 1, ContainsCost: 1, Size: refs.Size{ Value: int64(len(it.order)), Exact: true, }, }, nil } // A Resolver iterator consists of it's order, an index (where it is in the, // process of iterating) and a store to resolve values from. type resolverNext struct { qs refs.Namer order []quad.Value values []refs.Ref cached bool index int err error result refs.Ref } // Creates a new Resolver iterator. func newResolverNext(qs refs.Namer, nodes []quad.Value) *resolverNext { it := &resolverNext{ qs: qs, order: make([]quad.Value, len(nodes)), } copy(it.order, nodes) return it } func (it *resolverNext) Close() error { return nil } func (it *resolverNext) TagResults(dst map[string]refs.Ref) {} func (it *resolverNext) String() string { return fmt.Sprintf("ResolverNext(%v, %v)", it.order, it.values) } // Resolve nodes to values func (it *resolverNext) resolve(ctx context.Context) error { values, err := refs.RefsOf(ctx, it.qs, it.order) if err != nil { return err } it.values = make([]refs.Ref, len(it.order)) copy(it.values, values) it.order = nil it.cached = true return nil } // Next advances the iterator. func (it *resolverNext) Next(ctx context.Context) bool { if !it.cached { it.err = it.resolve(ctx) if it.err != nil { return false } } if it.index >= len(it.values) { it.result = nil return false } it.result = it.values[it.index] it.index++ return true } func (it *resolverNext) Err() error { return it.err } func (it *resolverNext) Result() refs.Ref { return it.result } func (it *resolverNext) NextPath(ctx context.Context) bool { return false } // A Resolver iterator consists of it's order, an index (where it is in the, // process of iterating) and a store to resolve values from. type resolverContains struct { qs refs.Namer order []quad.Value nodes map[interface{}]quad.Value cached bool err error result refs.Ref } // Creates a new Resolver iterator. func newResolverContains(qs refs.Namer, nodes []quad.Value) *resolverContains { it := &resolverContains{ qs: qs, order: make([]quad.Value, len(nodes)), } copy(it.order, nodes) return it } func (it *resolverContains) Close() error { return nil } func (it *resolverContains) TagResults(dst map[string]refs.Ref) {} func (it *resolverContains) String() string { return fmt.Sprintf("ResolverContains(%v, %v)", it.order, it.nodes) } // Resolve nodes to values func (it *resolverContains) resolve(ctx context.Context) error { values, err := refs.RefsOf(ctx, it.qs, it.order) if err != nil { return err } // Generally there are going to be no/few duplicates given // so allocate maps large enough to accommodate all it.nodes = make(map[interface{}]quad.Value, len(it.order)) for index, value := range values { node := it.order[index] it.nodes[value.Key()] = node } it.order = nil it.cached = true return nil } // Check if the passed value is equal to one of the order stored in the iterator. func (it *resolverContains) Contains(ctx context.Context, value refs.Ref) bool { if !it.cached { it.err = it.resolve(ctx) if it.err != nil { return false } } _, ok := it.nodes[value.Key()] if ok { it.result = value } return ok } func (it *resolverContains) Err() error { return it.err } func (it *resolverContains) Result() refs.Ref { return it.result } func (it *resolverContains) NextPath(ctx context.Context) bool { return false } ================================================ FILE: graph/iterator/resolver_test.go ================================================ package iterator_test import ( "context" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph/graphmock" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) func TestResolverIteratorIterate(t *testing.T) { var ctx context.Context nodes := []quad.Value{ quad.String("1"), quad.String("2"), quad.String("3"), quad.String("4"), quad.String("5"), quad.String("3"), // Assert iterator can handle duplicate values } data := make([]quad.Quad, 0, len(nodes)) for _, node := range nodes { data = append(data, quad.Make(quad.String("0"), "has", node, nil)) } qs := &graphmock.Store{ Data: data, } expected := make(map[quad.Value]refs.Ref) for _, node := range nodes { var err error expected[node], err = qs.ValueOf(node) require.NoError(t, err) } it := iterator.NewResolver(qs, nodes...).Iterate() for _, node := range nodes { require.True(t, it.Next(ctx)) require.NoError(t, it.Err()) require.Equal(t, expected[node], it.Result()) } require.False(t, it.Next(ctx)) require.Nil(t, it.Result()) } func TestResolverIteratorNotFoundError(t *testing.T) { var ctx context.Context nodes := []quad.Value{ quad.String("1"), quad.String("2"), quad.String("3"), quad.String("4"), quad.String("5"), } data := make([]quad.Quad, 0) skip := 3 for i, node := range nodes { // Simulate a missing subject if i == skip { continue } data = append(data, quad.Make(quad.String("0"), "has", node, nil)) } qs := &graphmock.Store{ Data: data, } count := 0 it := iterator.NewResolver(qs, nodes...).Iterate() for it.Next(ctx) { count++ } require.Equal(t, 0, count) require.Error(t, it.Err()) require.Nil(t, it.Result()) } func TestResolverIteratorContains(t *testing.T) { tests := []struct { name string nodes []quad.Value subject quad.Value contains bool }{ { "contains", []quad.Value{ quad.String("1"), quad.String("2"), quad.String("3"), }, quad.String("2"), true, }, { "not contains", []quad.Value{ quad.String("1"), quad.String("3"), }, quad.String("2"), false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ctx context.Context data := make([]quad.Quad, 0, len(test.nodes)) for _, node := range test.nodes { data = append(data, quad.Make(quad.String("0"), "has", node, nil)) } qs := &graphmock.Store{ Data: data, } it := iterator.NewResolver(qs, test.nodes...).Lookup() require.Equal(t, test.contains, it.Contains(ctx, refs.PreFetched(test.subject))) }) } } ================================================ FILE: graph/iterator/save.go ================================================ package iterator import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" ) var ( _ TaggerBase = (*Save)(nil) ) func Tag(it Shape, tag string) Shape { if s, ok := it.(TaggerShape); ok { s.AddTags(tag) return s } else if s, ok := it.(TaggerShape); ok { s.AddTags(tag) return s } return NewSave(it, tag) } var ( _ Shape = (*Save)(nil) _ TaggerShape = (*Save)(nil) ) func NewSave(on Shape, tags ...string) *Save { s := &Save{it: on} s.AddTags(tags...) return s } type Save struct { it Shape tags []string fixedTags map[string]refs.Ref } func (it *Save) Iterate() Scanner { return newSaveNext(it.it.Iterate(), it.tags, it.fixedTags) } func (it *Save) Lookup() Index { return newSaveContains(it.it.Lookup(), it.tags, it.fixedTags) } func (it *Save) String() string { return fmt.Sprintf("Save(%v, %v)", it.tags, it.fixedTags) } // Add a tag to the iterator. func (it *Save) AddTags(tag ...string) { it.tags = append(it.tags, tag...) } func (it *Save) AddFixedTag(tag string, value refs.Ref) { if it.fixedTags == nil { it.fixedTags = make(map[string]refs.Ref) } it.fixedTags[tag] = value } // Tags returns the tags held in the tagger. The returned value must not be mutated. func (it *Save) Tags() []string { return it.tags } // Fixed returns the fixed tags held in the tagger. The returned value must not be mutated. func (it *Save) FixedTags() map[string]refs.Ref { return it.fixedTags } func (it *Save) CopyFromTagger(st TaggerBase) { it.tags = append(it.tags, st.Tags()...) fixed := st.FixedTags() if len(fixed) == 0 { return } if it.fixedTags == nil { it.fixedTags = make(map[string]refs.Ref, len(fixed)) } for k, v := range fixed { it.fixedTags[k] = v } } func (it *Save) Stats(ctx context.Context) (Costs, error) { return it.it.Stats(ctx) } func (it *Save) Optimize(ctx context.Context) (nit Shape, no bool) { sub, ok := it.it.Optimize(ctx) if len(it.tags) == 0 && len(it.fixedTags) == 0 { return sub, true } if st, ok2 := sub.(TaggerShape); ok2 { st.CopyFromTagger(it) return st, true } if !ok { return it, false } s := NewSave(sub) s.CopyFromTagger(it) return s, true } func (it *Save) SubIterators() []Shape { return []Shape{it.it} } func newSaveNext(it Scanner, tags []string, fixed map[string]refs.Ref) *saveNext { return &saveNext{it: it, tags: tags, fixedTags: fixed} } type saveNext struct { it Scanner tags []string fixedTags map[string]refs.Ref } func (it *saveNext) String() string { return fmt.Sprintf("Save(%v, %v)", it.tags, it.fixedTags) } func (it *saveNext) TagResults(dst map[string]refs.Ref) { it.it.TagResults(dst) v := it.Result() for _, tag := range it.tags { dst[tag] = v } for tag, value := range it.fixedTags { dst[tag] = value } } func (it *saveNext) Result() refs.Ref { return it.it.Result() } func (it *saveNext) Next(ctx context.Context) bool { return it.it.Next(ctx) } func (it *saveNext) NextPath(ctx context.Context) bool { return it.it.NextPath(ctx) } func (it *saveNext) Err() error { return it.it.Err() } func (it *saveNext) Close() error { return it.it.Close() } func newSaveContains(it Index, tags []string, fixed map[string]refs.Ref) *saveContains { return &saveContains{it: it, tags: tags, fixed: fixed} } type saveContains struct { it Index tags []string fixed map[string]refs.Ref } func (it *saveContains) String() string { return fmt.Sprintf("SaveContains(%v, %v)", it.tags, it.fixed) } func (it *saveContains) TagResults(dst map[string]refs.Ref) { it.it.TagResults(dst) v := it.Result() for _, tag := range it.tags { dst[tag] = v } for tag, value := range it.fixed { dst[tag] = value } } func (it *saveContains) Result() refs.Ref { return it.it.Result() } func (it *saveContains) NextPath(ctx context.Context) bool { return it.it.NextPath(ctx) } func (it *saveContains) Contains(ctx context.Context, v refs.Ref) bool { return it.it.Contains(ctx, v) } func (it *saveContains) Err() error { return it.it.Err() } func (it *saveContains) Close() error { return it.it.Close() } ================================================ FILE: graph/iterator/skip.go ================================================ package iterator import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" ) // Skip iterator will skip certain number of values from primary iterator. type Skip struct { skip int64 primaryIt Shape } func NewSkip(primaryIt Shape, off int64) *Skip { return &Skip{ skip: off, primaryIt: primaryIt, } } func (it *Skip) Iterate() Scanner { return newSkipNext(it.primaryIt.Iterate(), it.skip) } func (it *Skip) Lookup() Index { return newSkipContains(it.primaryIt.Lookup(), it.skip) } // SubIterators returns a slice of the sub iterators. func (it *Skip) SubIterators() []Shape { return []Shape{it.primaryIt} } func (it *Skip) Optimize(ctx context.Context) (Shape, bool) { optimizedPrimaryIt, optimized := it.primaryIt.Optimize(ctx) if it.skip == 0 { // nothing to skip return optimizedPrimaryIt, true } it.primaryIt = optimizedPrimaryIt return it, optimized } func (it *Skip) Stats(ctx context.Context) (Costs, error) { primaryStats, err := it.primaryIt.Stats(ctx) if primaryStats.Size.Exact { primaryStats.Size.Value -= it.skip if primaryStats.Size.Value < 0 { primaryStats.Size.Value = 0 } } return primaryStats, err } func (it *Skip) String() string { return fmt.Sprintf("Skip(%d)", it.skip) } // Skip iterator will skip certain number of values from primary iterator. type skipNext struct { skip int64 skipped int64 primaryIt Scanner } func newSkipNext(primaryIt Scanner, skip int64) *skipNext { return &skipNext{ skip: skip, primaryIt: primaryIt, } } func (it *skipNext) TagResults(dst map[string]refs.Ref) { it.primaryIt.TagResults(dst) } // Next advances the Skip iterator. It will skip all initial values // before returning actual result. func (it *skipNext) Next(ctx context.Context) bool { for ; it.skipped < it.skip; it.skipped++ { if !it.primaryIt.Next(ctx) { return false } } if it.primaryIt.Next(ctx) { return true } return false } func (it *skipNext) Err() error { return it.primaryIt.Err() } func (it *skipNext) Result() refs.Ref { return it.primaryIt.Result() } // NextPath checks whether there is another path. It will skip first paths // according to iterator parameter. func (it *skipNext) NextPath(ctx context.Context) bool { for ; it.skipped < it.skip; it.skipped++ { if !it.primaryIt.NextPath(ctx) { return false } } return it.primaryIt.NextPath(ctx) } // Close closes the primary and all iterators. It closes all subiterators // it can, but returns the first error it encounters. func (it *skipNext) Close() error { return it.primaryIt.Close() } func (it *skipNext) String() string { return fmt.Sprintf("SkipNext(%d)", it.skip) } // Skip iterator will skip certain number of values from primary iterator. type skipContains struct { skip int64 skipped int64 primaryIt Index } func newSkipContains(primaryIt Index, skip int64) *skipContains { return &skipContains{ skip: skip, primaryIt: primaryIt, } } func (it *skipContains) TagResults(dst map[string]refs.Ref) { it.primaryIt.TagResults(dst) } func (it *skipContains) Err() error { return it.primaryIt.Err() } func (it *skipContains) Result() refs.Ref { return it.primaryIt.Result() } func (it *skipContains) Contains(ctx context.Context, val refs.Ref) bool { inNextPath := false for it.skipped <= it.skip { // skipping main iterator results inNextPath = false if !it.primaryIt.Contains(ctx, val) { return false } it.skipped++ // TODO(dennwc): we don't really know if we should call NextPath or not, // and there is no good way to know if it.skipped <= it.skip { // skipping NextPath results inNextPath = true if !it.primaryIt.NextPath(ctx) { // main path exists, but we skipped it // and we skipped all alternative paths now // so we definitely "don't have" this value return false } it.skipped++ for it.skipped <= it.skip { if !it.primaryIt.NextPath(ctx) { return false } it.skipped++ } } } if inNextPath && it.primaryIt.NextPath(ctx) { return true } return it.primaryIt.Contains(ctx, val) } // NextPath checks whether there is another path. It will skip first paths // according to iterator parameter. func (it *skipContains) NextPath(ctx context.Context) bool { for ; it.skipped < it.skip; it.skipped++ { if !it.primaryIt.NextPath(ctx) { return false } } return it.primaryIt.NextPath(ctx) } // Close closes the primary and all iterators. It closes all subiterators // it can, but returns the first error it encounters. func (it *skipContains) Close() error { return it.primaryIt.Close() } func (it *skipContains) String() string { return fmt.Sprintf("SkipContains(%d)", it.skip) } ================================================ FILE: graph/iterator/skip_test.go ================================================ package iterator_test import ( "context" "testing" "github.com/stretchr/testify/require" . "github.com/cayleygraph/cayley/graph/iterator" ) func TestSkipIteratorBasics(t *testing.T) { ctx := context.TODO() allIt := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), Int64Node(4), Int64Node(5), ) u := NewSkip(allIt, 0) expectSz, _ := allIt.Stats(ctx) sz, _ := u.Stats(ctx) require.Equal(t, expectSz.Size.Value, sz.Size.Value) require.Equal(t, []int{1, 2, 3, 4, 5}, iterated(u)) u = NewSkip(allIt, 3) expectSz.Size.Value = 2 if sz, _ := u.Stats(ctx); sz.Size.Value != expectSz.Size.Value { t.Errorf("Failed to check Skip size: got:%v expected:%v", sz.Size, expectSz.Size) } require.Equal(t, []int{4, 5}, iterated(u)) uc := u.Lookup() for _, v := range []int{1, 2, 3} { require.False(t, uc.Contains(ctx, Int64Node(v))) } for _, v := range []int{4, 5} { require.True(t, uc.Contains(ctx, Int64Node(v))) } uc = u.Lookup() for _, v := range []int{5, 4, 3} { require.False(t, uc.Contains(ctx, Int64Node(v))) } for _, v := range []int{1, 2} { require.True(t, uc.Contains(ctx, Int64Node(v))) } // TODO(dennwc): check with NextPath } ================================================ FILE: graph/iterator/sort.go ================================================ package iterator import ( "context" "sort" "github.com/cayleygraph/cayley/graph/refs" ) // Sort iterator orders values from it's subiterator. type Sort struct { namer refs.Namer subIt Shape } // NewSort creates a new Sort iterator. // TODO(dennwc): This iterator must not be used inside And: it may be moved to a Contains branch and won't do anything. // We should make And/Intersect account for this. func NewSort(namer refs.Namer, subIt Shape) *Sort { return &Sort{namer, subIt} } func (it *Sort) Iterate() Scanner { return newSortNext(it.namer, it.subIt.Iterate()) } func (it *Sort) Lookup() Index { // TODO(dennwc): Lookup doesn't need any sorting. Using it this way is a bug in the optimizer. // But instead of failing here, let still allow the query to execute. It won't be sorted, // but it will work at least. Later consider changing returning an error here. return it.subIt.Lookup() } func (it *Sort) Optimize(ctx context.Context) (Shape, bool) { newIt, optimized := it.subIt.Optimize(ctx) if optimized { it.subIt = newIt } return it, false } func (it *Sort) Stats(ctx context.Context) (Costs, error) { subStats, err := it.subIt.Stats(ctx) return Costs{ // TODO(dennwc): better cost calculation; we probably need an InitCost defined in Costs NextCost: subStats.NextCost * 2, ContainsCost: subStats.ContainsCost, Size: refs.Size{ Value: subStats.Size.Value, Exact: true, }, }, err } func (it *Sort) String() string { return "Sort" } // SubIterators returns a slice of the sub iterators. func (it *Sort) SubIterators() []Shape { return []Shape{it.subIt} } type sortValue struct { result str string paths []result } type sortByString []sortValue func (v sortByString) Len() int { return len(v) } func (v sortByString) Less(i, j int) bool { return v[i].str < v[j].str } func (v sortByString) Swap(i, j int) { v[i], v[j] = v[j], v[i] } type sortNext struct { namer refs.Namer subIt Scanner ordered sortByString result result err error index int pathIndex int } func newSortNext(namer refs.Namer, subIt Scanner) *sortNext { return &sortNext{ namer: namer, subIt: subIt, pathIndex: -1, } } func (it *sortNext) TagResults(dst map[string]refs.Ref) { for tag, value := range it.result.tags { dst[tag] = value } } func (it *sortNext) Err() error { return it.err } func (it *sortNext) Result() refs.Ref { return it.result.id } func (it *sortNext) Next(ctx context.Context) bool { if it.err != nil { return false } if it.ordered == nil { v, err := getSortedValues(ctx, it.namer, it.subIt) it.ordered = v it.err = err if it.err != nil { return false } } if it.index >= len(it.ordered) { return false } it.pathIndex = -1 it.result = it.ordered[it.index].result it.index++ return true } func (it *sortNext) NextPath(ctx context.Context) bool { if it.index >= len(it.ordered) { return false } r := it.ordered[it.index] if it.pathIndex+1 >= len(r.paths) { return false } it.pathIndex++ it.result = r.paths[it.pathIndex] return true } func (it *sortNext) Close() error { it.ordered = nil return it.subIt.Close() } func (it *sortNext) String() string { return "SortNext" } func getSortedValues(ctx context.Context, namer refs.Namer, it Scanner) (sortByString, error) { var v sortByString for it.Next(ctx) { id := it.Result() // TODO(dennwc): batch and use refs.ValuesOf name, err := namer.NameOf(id) if err != nil { return nil, err } str := name.String() tags := make(map[string]refs.Ref) it.TagResults(tags) val := sortValue{ result: result{id, tags}, str: str, } for it.NextPath(ctx) { tags = make(map[string]refs.Ref) it.TagResults(tags) val.paths = append(val.paths, result{id, tags}) } v = append(v, val) } if err := it.Err(); err != nil { return v, err } sort.Sort(v) return v, nil } ================================================ FILE: graph/iterator/unique.go ================================================ package iterator import ( "context" "github.com/cayleygraph/cayley/graph/refs" ) // Unique iterator removes duplicate values from it's subiterator. type Unique struct { subIt Shape } func NewUnique(subIt Shape) *Unique { return &Unique{ subIt: subIt, } } func (it *Unique) Iterate() Scanner { return newUniqueNext(it.subIt.Iterate()) } func (it *Unique) Lookup() Index { return newUniqueContains(it.subIt.Lookup()) } // SubIterators returns a slice of the sub iterators. The first iterator is the // primary iterator, for which the complement is generated. func (it *Unique) SubIterators() []Shape { return []Shape{it.subIt} } func (it *Unique) Optimize(ctx context.Context) (Shape, bool) { newIt, optimized := it.subIt.Optimize(ctx) if optimized { it.subIt = newIt } return it, false } const uniquenessFactor = 2 func (it *Unique) Stats(ctx context.Context) (Costs, error) { subStats, err := it.subIt.Stats(ctx) return Costs{ NextCost: subStats.NextCost * uniquenessFactor, ContainsCost: subStats.ContainsCost, Size: refs.Size{ Value: subStats.Size.Value / uniquenessFactor, Exact: false, }, }, err } func (it *Unique) String() string { return "Unique" } // Unique iterator removes duplicate values from it's subiterator. type uniqueNext struct { subIt Scanner result refs.Ref err error seen map[interface{}]bool } func newUniqueNext(subIt Scanner) *uniqueNext { return &uniqueNext{ subIt: subIt, seen: make(map[interface{}]bool), } } func (it *uniqueNext) TagResults(dst map[string]refs.Ref) { if it.subIt != nil { it.subIt.TagResults(dst) } } // Next advances the subiterator, continuing until it returns a value which it // has not previously seen. func (it *uniqueNext) Next(ctx context.Context) bool { for it.subIt.Next(ctx) { curr := it.subIt.Result() key := refs.ToKey(curr) if ok := it.seen[key]; !ok { it.result = curr it.seen[key] = true return true } } it.err = it.subIt.Err() return false } func (it *uniqueNext) Err() error { return it.err } func (it *uniqueNext) Result() refs.Ref { return it.result } // NextPath for unique always returns false. If we were to return multiple // paths, we'd no longer be a unique result, so we have to choose only the first // path that got us here. Unique is serious on this point. func (it *uniqueNext) NextPath(ctx context.Context) bool { return false } // Close closes the primary iterators. func (it *uniqueNext) Close() error { it.seen = nil return it.subIt.Close() } func (it *uniqueNext) String() string { return "UniqueNext" } // Unique iterator removes duplicate values from it's subiterator. type uniqueContains struct { subIt Index } func newUniqueContains(subIt Index) *uniqueContains { return &uniqueContains{ subIt: subIt, } } func (it *uniqueContains) TagResults(dst map[string]refs.Ref) { if it.subIt != nil { it.subIt.TagResults(dst) } } func (it *uniqueContains) Err() error { return it.subIt.Err() } func (it *uniqueContains) Result() refs.Ref { return it.subIt.Result() } // Contains checks whether the passed value is part of the primary iterator, // which is irrelevant for uniqueness. func (it *uniqueContains) Contains(ctx context.Context, val refs.Ref) bool { return it.subIt.Contains(ctx, val) } // NextPath for unique always returns false. If we were to return multiple // paths, we'd no longer be a unique result, so we have to choose only the first // path that got us here. Unique is serious on this point. func (it *uniqueContains) NextPath(ctx context.Context) bool { return false } // Close closes the primary iterators. func (it *uniqueContains) Close() error { return it.subIt.Close() } func (it *uniqueContains) String() string { return "UniqueContains" } ================================================ FILE: graph/iterator/unique_test.go ================================================ package iterator_test import ( "context" "testing" "github.com/stretchr/testify/require" . "github.com/cayleygraph/cayley/graph/iterator" ) func TestUniqueIteratorBasics(t *testing.T) { ctx := context.TODO() allIt := NewFixed( Int64Node(1), Int64Node(2), Int64Node(3), Int64Node(3), Int64Node(2), ) u := NewUnique(allIt) expect := []int{1, 2, 3} for i := 0; i < 2; i++ { require.Equal(t, expect, iterated(u)) } uc := u.Lookup() for _, v := range []int{1, 2, 3} { require.True(t, uc.Contains(ctx, Int64Node(v))) } } ================================================ FILE: graph/iterator/value_comparison.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator // "Value Comparison" is a unary operator -- a filter across the values in the // relevant subiterator. // // This is hugely useful for things like label, but value ranges in general // come up from time to time. At *worst* we're as big as our underlying iterator. // At best, we're the null iterator. // // This is ripe for backend-side optimization. If you can run a value iterator, // from a sorted set -- some sort of value index, then go for it. // // In MQL terms, this is the [{"age>=": 21}] concept. import ( "fmt" "time" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) type Operator int func (op Operator) String() string { switch op { case CompareLT: return "<" case CompareLTE: return "<=" case CompareGT: return ">" case CompareGTE: return ">=" default: return fmt.Sprintf("op(%d)", int(op)) } } const ( CompareLT Operator = iota CompareLTE CompareGT CompareGTE // Why no Equals? Because that's usually an AndIterator. ) func NewComparison(sub Shape, op Operator, val quad.Value, qs refs.Namer) Shape { return NewValueFilter(qs, sub, func(qval quad.Value) (bool, error) { switch cVal := val.(type) { case quad.Int: if cVal2, ok := qval.(quad.Int); ok { return RunIntOp(cVal2, op, cVal), nil } return false, nil case quad.Float: if cVal2, ok := qval.(quad.Float); ok { return RunFloatOp(cVal2, op, cVal), nil } return false, nil case quad.String: if cVal2, ok := qval.(quad.String); ok { return RunStrOp(string(cVal2), op, string(cVal)), nil } return false, nil case quad.BNode: if cVal2, ok := qval.(quad.BNode); ok { return RunStrOp(string(cVal2), op, string(cVal)), nil } return false, nil case quad.IRI: if cVal2, ok := qval.(quad.IRI); ok { return RunStrOp(string(cVal2), op, string(cVal)), nil } return false, nil case quad.Time: if cVal2, ok := qval.(quad.Time); ok { return RunTimeOp(time.Time(cVal2), op, time.Time(cVal)), nil } return false, nil default: return RunStrOp(quad.StringOf(qval), op, quad.StringOf(val)), nil } }) } func RunIntOp(a quad.Int, op Operator, b quad.Int) bool { switch op { case CompareLT: return a < b case CompareLTE: return a <= b case CompareGT: return a > b case CompareGTE: return a >= b default: panic("Unknown operator type") } } func RunFloatOp(a quad.Float, op Operator, b quad.Float) bool { switch op { case CompareLT: return a < b case CompareLTE: return a <= b case CompareGT: return a > b case CompareGTE: return a >= b default: panic("Unknown operator type") } } func RunStrOp(a string, op Operator, b string) bool { switch op { case CompareLT: return a < b case CompareLTE: return a <= b case CompareGT: return a > b case CompareGTE: return a >= b default: panic("Unknown operator type") } } func RunTimeOp(a time.Time, op Operator, b time.Time) bool { switch op { case CompareLT: return a.Before(b) case CompareLTE: return !a.After(b) case CompareGT: return a.After(b) case CompareGTE: return !a.Before(b) default: panic("Unknown operator type") } } ================================================ FILE: graph/iterator/value_comparison_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator_test import ( "context" "errors" "reflect" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph/graphmock" . "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) var ( simpleStore = &graphmock.Oldstore{Data: []string{"0", "1", "2", "3", "4", "5"}, Parse: true} stringStore = &graphmock.Oldstore{Data: []string{"foo", "bar", "baz", "echo"}, Parse: true} mixedStore = &graphmock.Oldstore{Data: []string{"0", "1", "2", "3", "4", "5", "foo", "bar", "baz", "echo"}, Parse: true} ) func simpleFixedIterator() *Fixed { f := NewFixed() for i := 0; i < 5; i++ { f.Add(Int64Node(i)) } return f } func stringFixedIterator() *Fixed { f := NewFixed() for _, value := range stringStore.Data { f.Add(graphmock.StringNode(value)) } return f } func mixedFixedIterator() *Fixed { f := NewFixed() for i := 0; i < len(mixedStore.Data); i++ { f.Add(Int64Node(i)) } return f } var comparisonTests = []struct { message string operand quad.Value operator Operator expect []quad.Value qs refs.Namer iterator func() *Fixed }{ { message: "successful int64 less than comparison", operand: quad.Int(3), operator: CompareLT, expect: []quad.Value{quad.Int(0), quad.Int(1), quad.Int(2)}, qs: simpleStore, iterator: simpleFixedIterator, }, { message: "empty int64 less than comparison", operand: quad.Int(0), operator: CompareLT, expect: nil, qs: simpleStore, iterator: simpleFixedIterator, }, { message: "successful int64 greater than comparison", operand: quad.Int(2), operator: CompareGT, expect: []quad.Value{quad.Int(3), quad.Int(4)}, qs: simpleStore, iterator: simpleFixedIterator, }, { message: "successful int64 greater than or equal comparison", operand: quad.Int(2), operator: CompareGTE, expect: []quad.Value{quad.Int(2), quad.Int(3), quad.Int(4)}, qs: simpleStore, iterator: simpleFixedIterator, }, { message: "successful int64 greater than or equal comparison (mixed)", operand: quad.Int(2), operator: CompareGTE, expect: []quad.Value{quad.Int(2), quad.Int(3), quad.Int(4), quad.Int(5)}, qs: mixedStore, iterator: mixedFixedIterator, }, { message: "successful string less than comparison", operand: quad.String("echo"), operator: CompareLT, expect: []quad.Value{quad.String("bar"), quad.String("baz")}, qs: stringStore, iterator: stringFixedIterator, }, { message: "empty string less than comparison", operand: quad.String(""), operator: CompareLT, expect: nil, qs: stringStore, iterator: stringFixedIterator, }, { message: "successful string greater than comparison", operand: quad.String("echo"), operator: CompareGT, expect: []quad.Value{quad.String("foo")}, qs: stringStore, iterator: stringFixedIterator, }, { message: "successful string greater than or equal comparison", operand: quad.String("echo"), operator: CompareGTE, expect: []quad.Value{quad.String("foo"), quad.String("echo")}, qs: stringStore, iterator: stringFixedIterator, }, } func TestValueComparison(t *testing.T) { ctx := context.TODO() for _, test := range comparisonTests { qs := test.qs vc := NewComparison(test.iterator(), test.operator, test.operand, qs).Iterate() var got []quad.Value for vc.Next(ctx) { qsv, err := qs.NameOf(vc.Result()) require.NoError(t, err) got = append(got, qsv) } if !reflect.DeepEqual(got, test.expect) { t.Errorf("Failed to show %s, got:%q expect:%q", test.message, got, test.expect) } } } var vciContainsTests = []struct { message string operator Operator check refs.Ref expect bool qs refs.Namer val quad.Value iterator func() *Fixed }{ { message: "1 is less than 2", operator: CompareGTE, check: Int64Node(1), expect: false, qs: simpleStore, val: quad.Int(2), iterator: simpleFixedIterator, }, { message: "2 is greater than or equal to 2", operator: CompareGTE, check: Int64Node(2), expect: true, qs: simpleStore, val: quad.Int(2), iterator: simpleFixedIterator, }, { message: "3 is greater than or equal to 2", operator: CompareGTE, check: Int64Node(3), expect: true, qs: simpleStore, val: quad.Int(2), iterator: simpleFixedIterator, }, { message: "5 is absent from iterator", operator: CompareGTE, check: Int64Node(5), expect: false, qs: simpleStore, val: quad.Int(2), iterator: simpleFixedIterator, }, { message: "foo is greater than or equal to echo", operator: CompareGTE, check: graphmock.StringNode("foo"), expect: true, qs: stringStore, val: quad.String("echo"), iterator: stringFixedIterator, }, { message: "echo is greater than or equal to echo", operator: CompareGTE, check: graphmock.StringNode("echo"), expect: true, qs: stringStore, val: quad.String("echo"), iterator: stringFixedIterator, }, { message: "foo is missing from the iterator", operator: CompareLTE, check: graphmock.StringNode("foo"), expect: false, qs: stringStore, val: quad.String("echo"), iterator: stringFixedIterator, }, } func TestVCIContains(t *testing.T) { ctx := context.TODO() for _, test := range vciContainsTests { vc := NewComparison(test.iterator(), test.operator, test.val, test.qs).Lookup() if vc.Contains(ctx, test.check) != test.expect { t.Errorf("Failed to show %s", test.message) } } } var comparisonIteratorTests = []struct { message string qs refs.Namer val quad.Value }{ { message: "2 is absent from iterator", qs: simpleStore, val: quad.Int(2), }, { message: "'missing' is absent from iterator", qs: stringStore, val: quad.String("missing"), }, } func TestComparisonIteratorErr(t *testing.T) { ctx := context.TODO() wantErr := errors.New("unique") errIt := newTestIterator(false, wantErr) for _, test := range comparisonIteratorTests { vc := NewComparison(errIt, CompareLT, test.val, test.qs).Iterate() require.False(t, vc.Next(ctx)) require.Equal(t, wantErr, vc.Err()) } } ================================================ FILE: graph/iterator/value_filter.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package iterator import ( "context" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) type ValueFilterFunc func(quad.Value) (bool, error) type ValueFilter struct { sub Shape filter ValueFilterFunc qs refs.Namer } func NewValueFilter(qs refs.Namer, sub Shape, filter ValueFilterFunc) *ValueFilter { return &ValueFilter{ sub: sub, qs: qs, filter: filter, } } func (it *ValueFilter) Iterate() Scanner { return newValueFilterNext(it.qs, it.sub.Iterate(), it.filter) } func (it *ValueFilter) Lookup() Index { return newValueFilterContains(it.qs, it.sub.Lookup(), it.filter) } func (it *ValueFilter) SubIterators() []Shape { return []Shape{it.sub} } func (it *ValueFilter) String() string { return "ValueFilter" } // There's nothing to optimize, locally, for a value-comparison iterator. // Replace the underlying iterator if need be. // potentially replace it. func (it *ValueFilter) Optimize(ctx context.Context) (Shape, bool) { newSub, changed := it.sub.Optimize(ctx) if changed { it.sub = newSub } return it, true } // We're only as expensive as our subiterator. // Again, optimized value comparison iterators should do better. func (it *ValueFilter) Stats(ctx context.Context) (Costs, error) { st, err := it.sub.Stats(ctx) st.Size.Value = st.Size.Value/2 + 1 st.Size.Exact = false return st, err } type valueFilterNext struct { sub Scanner filter ValueFilterFunc qs refs.Namer result refs.Ref err error } func newValueFilterNext(qs refs.Namer, sub Scanner, filter ValueFilterFunc) *valueFilterNext { return &valueFilterNext{ sub: sub, qs: qs, filter: filter, } } func (it *valueFilterNext) doFilter(val refs.Ref) bool { qval, err := it.qs.NameOf(val) if err != nil { it.err = err return false } ok, err := it.filter(qval) if err != nil { it.err = err } return ok } func (it *valueFilterNext) Close() error { return it.sub.Close() } func (it *valueFilterNext) Next(ctx context.Context) bool { for it.sub.Next(ctx) { val := it.sub.Result() if it.doFilter(val) { it.result = val return true } } it.err = it.sub.Err() return false } func (it *valueFilterNext) Err() error { return it.err } func (it *valueFilterNext) Result() refs.Ref { return it.result } func (it *valueFilterNext) NextPath(ctx context.Context) bool { return it.sub.NextPath(ctx) } // If we failed the check, then the subiterator should not contribute to the result // set. Otherwise, go ahead and tag it. func (it *valueFilterNext) TagResults(dst map[string]refs.Ref) { it.sub.TagResults(dst) } func (it *valueFilterNext) String() string { return "ValueFilterNext" } type valueFilterContains struct { sub Index filter ValueFilterFunc qs refs.Namer result refs.Ref err error } func newValueFilterContains(qs refs.Namer, sub Index, filter ValueFilterFunc) *valueFilterContains { return &valueFilterContains{ sub: sub, qs: qs, filter: filter, } } func (it *valueFilterContains) doFilter(val refs.Ref) bool { qval, err := it.qs.NameOf(val) if err != nil { it.err = err return false } ok, err := it.filter(qval) if err != nil { it.err = err } return ok } func (it *valueFilterContains) Close() error { return it.sub.Close() } func (it *valueFilterContains) Err() error { return it.err } func (it *valueFilterContains) Result() refs.Ref { return it.result } func (it *valueFilterContains) NextPath(ctx context.Context) bool { return it.sub.NextPath(ctx) } func (it *valueFilterContains) Contains(ctx context.Context, val refs.Ref) bool { if !it.doFilter(val) { return false } ok := it.sub.Contains(ctx, val) if !ok { it.err = it.sub.Err() } return ok } // If we failed the check, then the subiterator should not contribute to the result // set. Otherwise, go ahead and tag it. func (it *valueFilterContains) TagResults(dst map[string]refs.Ref) { it.sub.TagResults(dst) } func (it *valueFilterContains) String() string { return "ValueFilterContains" } ================================================ FILE: graph/kv/all/all.go ================================================ package all import ( // import all implementations that hidalgo supports _ "github.com/hidal-go/hidalgo/kv/all" // make sure to import kv package, so it can re-register hidalgo's backends _ "github.com/cayleygraph/cayley/graph/kv" // legacy: override bolt implementation; check the package for details _ "github.com/cayleygraph/cayley/graph/kv/bolt" ) ================================================ FILE: graph/kv/all_iterator.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kv import ( "context" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/proto" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) type constraint struct { dir quad.Direction val Int64Value } type allIterator struct { qs *QuadStore nodes bool cons *constraint } func (qs *QuadStore) newAllIterator(nodes bool, cons *constraint) *allIterator { if nodes && cons != nil { panic("cannot use a kv all iterator across nodes with a constraint") } return &allIterator{ qs: qs, nodes: nodes, cons: cons, } } func (it *allIterator) Iterate() iterator.Scanner { return it.qs.newAllIteratorNext(it.nodes, it.cons) } func (it *allIterator) Lookup() iterator.Index { return it.qs.newAllIteratorContains(it.nodes, it.cons) } // No subiterators. func (it *allIterator) SubIterators() []iterator.Shape { return nil } func (it *allIterator) String() string { return "KVAll" } func (it *allIterator) Sorted() bool { return false } func (it *allIterator) Optimize(ctx context.Context) (iterator.Shape, bool) { return it, false } func (it *allIterator) Stats(ctx context.Context) (iterator.Costs, error) { return iterator.Costs{ ContainsCost: 1, NextCost: 2, Size: refs.Size{ Value: it.qs.Size(), Exact: false, }, }, nil } type allIteratorNext struct { nodes bool id uint64 buf []*proto.Primitive prim *proto.Primitive horizon int64 qs *QuadStore err error cons *constraint } func (qs *QuadStore) newAllIteratorNext(nodes bool, cons *constraint) *allIteratorNext { if nodes && cons != nil { panic("cannot use a kv all iterator across nodes with a constraint\n") } return &allIteratorNext{ qs: qs, nodes: nodes, horizon: qs.horizon(context.TODO()), cons: cons, } } func (it *allIteratorNext) TagResults(dst map[string]graph.Ref) {} func (it *allIteratorNext) Close() error { return nil } func (it *allIteratorNext) Err() error { return it.err } func (it *allIteratorNext) Result() graph.Ref { if it.id > uint64(it.horizon) { return nil } if it.nodes { return Int64Value(it.id) } if it.prim == nil { return nil } return it.prim } const nextBatch = 100 func (it *allIteratorNext) Next(ctx context.Context) bool { if it.err != nil { return false } for { if len(it.buf) == 0 { if it.id+1 > uint64(it.horizon) { return false } ids := make([]uint64, 0, nextBatch) for i := 0; i < nextBatch; i++ { it.id++ if it.id > uint64(it.horizon) { break } ids = append(ids, it.id) } if len(ids) == 0 { return false } it.buf, it.err = it.qs.getPrimitives(ctx, ids) if it.err != nil || len(it.buf) == 0 { return false } } else { it.buf = it.buf[1:] } for ; len(it.buf) > 0; it.buf = it.buf[1:] { p := it.buf[0] it.prim = p if p == nil || p.Deleted { continue } it.id = it.prim.ID if p.IsNode() && it.nodes { return true } if !p.IsNode() && !it.nodes { if it.cons == nil { return true } if Int64Value(p.GetDirection(it.cons.dir)) == it.cons.val { return true } } } } } func (it *allIteratorNext) NextPath(ctx context.Context) bool { return false } func (it *allIteratorNext) String() string { return "KVAllNext" } func (it *allIteratorNext) Sorted() bool { return false } type allIteratorContains struct { nodes bool id uint64 prim *proto.Primitive horizon int64 qs *QuadStore err error cons *constraint } func (qs *QuadStore) newAllIteratorContains(nodes bool, cons *constraint) *allIteratorContains { if nodes && cons != nil { panic("cannot use a kv all iterator across nodes with a constraint") } return &allIteratorContains{ qs: qs, nodes: nodes, horizon: qs.horizon(context.TODO()), cons: cons, } } func (it *allIteratorContains) TagResults(dst map[string]graph.Ref) {} func (it *allIteratorContains) Close() error { return nil } func (it *allIteratorContains) Err() error { return it.err } func (it *allIteratorContains) Result() graph.Ref { if it.id > uint64(it.horizon) { return nil } if it.nodes { return Int64Value(it.id) } if it.prim == nil { return nil } return it.prim } func (it *allIteratorContains) NextPath(ctx context.Context) bool { return false } func (it *allIteratorContains) Contains(ctx context.Context, v graph.Ref) bool { // TODO(dennwc): This method doesn't check if the primitive still exists in the store. // It's okay if we assume we provide the snapshot of data, though. // However, passing a hand-crafted Ref will cause invalid results. // Same is true for QuadIterator. if it.nodes { x, ok := v.(Int64Value) if !ok { return false } it.id = uint64(x) return it.id <= uint64(it.horizon) } p, ok := v.(*proto.Primitive) if !ok { return false } it.prim = p it.id = it.prim.ID if it.cons == nil { return true } if Int64Value(it.prim.GetDirection(it.cons.dir)) != it.cons.val { return false } return true } func (it *allIteratorContains) String() string { return "KVAllContains" } func (it *allIteratorContains) Sorted() bool { return false } ================================================ FILE: graph/kv/badger/badger.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package badger import ( "os" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv" hkv "github.com/hidal-go/hidalgo/kv" "github.com/hidal-go/hidalgo/kv/flat" "github.com/hidal-go/hidalgo/kv/flat/badger" ) const ( Type = badger.Name ) func Create(path string, m graph.Options) (hkv.KV, error) { if path == "" { return nil, kv.ErrEmptyPath } err := os.MkdirAll(path, 0700) if err != nil { return nil, err } db, err := badger.OpenPath(path) if err != nil { return nil, err } return flat.Upgrade(db), nil } ================================================ FILE: graph/kv/badger/badger_test.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package badger import ( "io/ioutil" "os" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv/kvtest" hkv "github.com/hidal-go/hidalgo/kv" ) func makeBadgerkv(t testing.TB) (hkv.KV, graph.Options, func()) { tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test_"+Type) if err != nil { t.Fatalf("Could not create working directory: %v", err) } db, err := Create(tmpDir, nil) if err != nil { os.RemoveAll(tmpDir) t.Fatal("Failed to create Badger database.", err) } return db, nil, func() { db.Close() os.RemoveAll(tmpDir) } } func TestBadgerkv(t *testing.T) { kvtest.TestAll(t, makeBadgerkv, nil) } func BenchmarkBadgerkv(b *testing.B) { kvtest.BenchmarkAll(b, makeBadgerkv, nil) } ================================================ FILE: graph/kv/bbolt/bolt.go ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bbolt import ( "os" "path/filepath" hkv "github.com/hidal-go/hidalgo/kv" bolt "github.com/hidal-go/hidalgo/kv/bbolt" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv" ) func init() { // override implementation; hidalgo expects a path to a database file, // while cayley was using path/index.bolt file previously kv.Register(Type, kv.Registration{ NewFunc: Open, InitFunc: Create, IsPersistent: true, }) } const ( Type = bolt.Name ) func getBoltFile(cfgpath string) string { return filepath.Join(cfgpath, "indexes.bolt") } func Create(path string, _ graph.Options) (hkv.KV, error) { if path == "" { return nil, kv.ErrEmptyPath } err := os.MkdirAll(path, 0700) if err != nil { return nil, err } db, err := bolt.Open(getBoltFile(path), nil) if err != nil { clog.Errorf("Error: couldn't create Bolt database: %v", err) return nil, err } return db, nil } func Open(path string, opt graph.Options) (hkv.KV, error) { db, err := bolt.Open(getBoltFile(path), nil) if err != nil { clog.Errorf("Error, couldn't open! %v", err) return nil, err } bdb := db.DB() // BoolKey returns false on non-existence. IE, Sync by default. bdb.NoSync, err = opt.BoolKey("nosync", false) if err != nil { db.Close() return nil, err } bdb.NoGrowSync = bdb.NoSync if bdb.NoSync { clog.Infof("Running in nosync mode") } return db, nil } ================================================ FILE: graph/kv/bbolt/bolt_test.go ================================================ // Copyright 2015 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bbolt import ( "io/ioutil" "os" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv/kvtest" hkv "github.com/hidal-go/hidalgo/kv" ) func makeBolt(t testing.TB) (hkv.KV, graph.Options, func()) { tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test_"+Type) if err != nil { t.Fatalf("Could not create working directory: %v", err) } db, err := Create(tmpDir, nil) if err != nil { os.RemoveAll(tmpDir) t.Fatal("Failed to create Bolt database.", err) } return db, nil, func() { db.Close() os.RemoveAll(tmpDir) } } func TestBolt(t *testing.T) { kvtest.TestAll(t, makeBolt, nil) } func BenchmarkBolt(b *testing.B) { kvtest.BenchmarkAll(b, makeBolt, nil) } ================================================ FILE: graph/kv/bolt/bolt.go ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bolt import ( "os" "path/filepath" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv" hkv "github.com/hidal-go/hidalgo/kv" "github.com/hidal-go/hidalgo/kv/bolt" ) func init() { // override implementation; hidalgo expects a path to a database file, // while cayley was using path/index.bolt file previously kv.Register(Type, kv.Registration{ NewFunc: Open, InitFunc: Create, IsPersistent: true, }) } const ( Type = bolt.Name ) func getBoltFile(cfgpath string) string { return filepath.Join(cfgpath, "indexes.bolt") } func Create(path string, _ graph.Options) (hkv.KV, error) { if path == "" { return nil, kv.ErrEmptyPath } err := os.MkdirAll(path, 0700) if err != nil { return nil, err } db, err := bolt.Open(getBoltFile(path), nil) if err != nil { clog.Errorf("Error: couldn't create Bolt database: %v", err) return nil, err } return db, nil } func Open(path string, opt graph.Options) (hkv.KV, error) { db, err := bolt.Open(getBoltFile(path), nil) if err != nil { clog.Errorf("Error, couldn't open! %v", err) return nil, err } bdb := db.DB() // BoolKey returns false on non-existence. IE, Sync by default. bdb.NoSync, err = opt.BoolKey("nosync", false) if err != nil { db.Close() return nil, err } bdb.NoGrowSync = bdb.NoSync if bdb.NoSync { clog.Infof("Running in nosync mode") } return db, nil } ================================================ FILE: graph/kv/bolt/bolt_test.go ================================================ // Copyright 2015 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bolt import ( "io/ioutil" "os" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv/kvtest" hkv "github.com/hidal-go/hidalgo/kv" ) func makeBolt(t testing.TB) (hkv.KV, graph.Options, func()) { tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test_"+Type) if err != nil { t.Fatalf("Could not create working directory: %v", err) } db, err := Create(tmpDir, nil) if err != nil { os.RemoveAll(tmpDir) t.Fatal("Failed to create Bolt database.", err) } return db, nil, func() { db.Close() os.RemoveAll(tmpDir) } } func TestBolt(t *testing.T) { kvtest.TestAll(t, makeBolt, nil) } func BenchmarkBolt(b *testing.B) { kvtest.BenchmarkAll(b, makeBolt, nil) } ================================================ FILE: graph/kv/btree/btree.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package btree import ( "github.com/cayleygraph/cayley/graph" hkv "github.com/hidal-go/hidalgo/kv" "github.com/hidal-go/hidalgo/kv/flat" "github.com/hidal-go/hidalgo/kv/flat/btree" ) const ( Type = btree.Name ) func Create(path string, _ graph.Options) (hkv.KV, error) { return New(), nil } func New() hkv.KV { return flat.Upgrade(btree.New()) } ================================================ FILE: graph/kv/btree/btree_test.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package btree import ( "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv/kvtest" hkv "github.com/hidal-go/hidalgo/kv" "github.com/hidal-go/hidalgo/kv/kvdebug" ) const debug = false func makeBtree(t testing.TB) (hkv.KV, graph.Options, func()) { if debug { return makeBtreeDebug(t) } return New(), nil, func() {} } func makeBtreeDebug(t testing.TB) (hkv.KV, graph.Options, func()) { db := New() d := kvdebug.New(db) d.Log(true) return d, nil, func() { d.Close() t.Logf("kv stats: %+v", d.Stats()) } } var conf = &kvtest.Config{ AlwaysRunIntegration: true, } func TestBtree(t *testing.T) { kvtest.TestAll(t, makeBtree, conf) } func BenchmarkBtree(b *testing.B) { kvtest.BenchmarkAll(b, makeBtree, conf) } ================================================ FILE: graph/kv/indexing.go ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kv import ( "context" "encoding/binary" "encoding/json" "errors" "fmt" "io" "sort" "time" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/pquads" "github.com/hidal-go/hidalgo/kv/options" "google.golang.org/protobuf/proto" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" graphlog "github.com/cayleygraph/cayley/graph/log" cproto "github.com/cayleygraph/cayley/graph/proto" "github.com/cayleygraph/cayley/graph/refs" "github.com/hidal-go/hidalgo/kv" "github.com/prometheus/client_golang/prometheus" boom "github.com/tylertreat/BoomFilters" ) var ( metaBucket = kv.Key{[]byte("meta")} logIndex = kv.Key{[]byte("log")} keyMetaIndexes = metaBucket.AppendBytes([]byte("indexes")) // List of all buckets in the current version of the database. buckets = []kv.Key{ metaBucket, logIndex, } // legacyQuadIndexes is a set of indexes used in Cayley < 0.7.6 legacyQuadIndexes = []QuadIndex{ {Dirs: []quad.Direction{quad.Subject}}, {Dirs: []quad.Direction{quad.Object}}, } DefaultQuadIndexes = []QuadIndex{ // First index optimizes forward traversals. Getting all relations for a node should // also be reasonably fast (prefix scan). {Dirs: []quad.Direction{quad.Subject, quad.Predicate}}, // Second index helps with reverse traversals as well as full quad lookups. // It also prevents issues with super-nodes, since most of those are values // with a high in-degree. {Dirs: []quad.Direction{quad.Object, quad.Predicate, quad.Subject}}, } ) var quadKeyEnc = binary.BigEndian type QuadIndex struct { Dirs []quad.Direction `json:"dirs"` Unique bool `json:"unique"` } func (ind QuadIndex) Key(vals []uint64) kv.Key { key := make([]byte, 8*len(vals)) n := 0 for i := range vals { quadKeyEnc.PutUint64(key[n:], vals[i]) n += 8 } // TODO(dennwc): split into parts? return ind.bucket().AppendBytes(key) } func (ind QuadIndex) KeyFor(p *cproto.Primitive) kv.Key { key := make([]byte, 8*len(ind.Dirs)) n := 0 for _, d := range ind.Dirs { quadKeyEnc.PutUint64(key[n:], p.GetDirection(d)) n += 8 } // TODO(dennwc): split into parts? return ind.bucket().AppendBytes(key) } func (ind QuadIndex) bucket() kv.Key { buf := make([]byte, len(ind.Dirs)) for i, d := range ind.Dirs { buf[i] = d.Prefix() } key := make(kv.Key, 1, 2) key[0] = buf return key } func bucketForVal(i, j byte) kv.Key { return kv.Key{[]byte{'v', i, j}} } func bucketForValRefs(i, j byte) kv.Key { return kv.Key{[]byte{'n', i, j}} } func (qs *QuadStore) createBuckets(ctx context.Context, upfront bool) error { err := kv.Update(ctx, qs.db, func(tx kv.Tx) error { for _, index := range buckets { _ = kv.CreateBucket(ctx, tx, index) } for _, ind := range qs.indexes.all { _ = kv.CreateBucket(ctx, tx, ind.bucket()) } return nil }) if err != nil { return err } if !upfront { return nil } for i := 0; i < 256; i++ { err := kv.Update(ctx, qs.db, func(tx kv.Tx) error { for j := 0; j < 256; j++ { _ = kv.CreateBucket(ctx, tx, bucketForVal(byte(i), byte(j))) _ = kv.CreateBucket(ctx, tx, bucketForValRefs(byte(i), byte(j))) } return nil }) if err != nil { return err } } return nil } func (qs *QuadStore) incSize(ctx context.Context, tx kv.Tx, size int64) error { _, err := qs.incMetaInt(ctx, tx, "size", size) return err } // writeIndexesMeta writes metadata about current indexes to the KV database, // so we can read this information back later. func (qs *QuadStore) writeIndexesMeta(ctx context.Context) error { // TODO(dennwc): change to protobuf later? data, err := json.Marshal(qs.indexes.all) if err != nil { return err } return kv.Update(ctx, qs.db, func(tx kv.Tx) error { return tx.Put(ctx, keyMetaIndexes, data) }) } // readIndexesMeta read metadata about current indexes from the KV database. // If no indexes are set, it returns a list of legacy indexes to preserve backward compatibility. func (qs *QuadStore) readIndexesMeta(ctx context.Context) ([]QuadIndex, error) { tx, err := qs.db.Tx(ctx, false) if err != nil { return nil, err } defer tx.Close() tx = wrapTx(tx) val, err := tx.Get(ctx, keyMetaIndexes) if err == kv.ErrNotFound { return legacyQuadIndexes, nil } else if err != nil { return nil, err } var out []QuadIndex if err := json.Unmarshal(val, &out); err != nil { return nil, fmt.Errorf("cannot decode indexes: %v", err) } else if len(out) == 0 { return legacyQuadIndexes, nil } return out, nil } func (qs *QuadStore) resolveValDeltas(ctx context.Context, tx kv.Tx, deltas []graphlog.NodeUpdate, fnc func(i int, id uint64)) error { inds := make([]int, 0, len(deltas)) keys := make([]kv.Key, 0, len(deltas)) for i, d := range deltas { if iri, ok := d.Val.(quad.IRI); ok { if x, ok := qs.valueLRU.Get(string(iri)); ok { fnc(i, x.(uint64)) continue } } else if d.Val == nil { fnc(i, 0) continue } if qs.mapNodes != nil && !qs.mapNodes.Test(d.Hash[:]) { fnc(i, 0) continue } inds = append(inds, i) keys = append(keys, bucketKeyForHash(d.Hash)) } if len(keys) == 0 { return nil } resp, err := tx.GetBatch(ctx, keys) if err != nil { return err } keys = nil for i, b := range resp { if len(b) == 0 { fnc(inds[i], 0) continue } ind := inds[i] id, _ := binary.Uvarint(b) d := &deltas[ind] if iri, ok := d.Val.(quad.IRI); ok && id != 0 { qs.valueLRU.Put(string(iri), uint64(id)) } fnc(ind, uint64(id)) } return nil } func (qs *QuadStore) getMetaIntTx(ctx context.Context, tx kv.Tx, key string) (int64, error) { val, err := tx.Get(ctx, metaBucket.AppendBytes([]byte(key))) if err == kv.ErrNotFound { return 0, err } else if err != nil { return 0, fmt.Errorf("cannot get horizon value: %v", err) } return int64(binary.LittleEndian.Uint64(val)), nil } func (qs *QuadStore) incMetaInt(ctx context.Context, tx kv.Tx, key string, n int64) (int64, error) { if n == 0 { return 0, nil } v, err := qs.getMetaIntTx(ctx, tx, key) if err != nil && err != kv.ErrNotFound { return 0, fmt.Errorf("cannot get %s: %v", key, err) } start := v v += n buf := make([]byte, 8) // bolt needs all slices available on Commit binary.LittleEndian.PutUint64(buf, uint64(v)) err = tx.Put(ctx, metaBucket.AppendBytes([]byte(key)), buf) if err != nil { return 0, fmt.Errorf("cannot inc %s: %v", key, err) } return start, nil } func (qs *QuadStore) genIDs(ctx context.Context, tx kv.Tx, n int) (uint64, error) { if n == 0 { return 0, nil } start, err := qs.incMetaInt(ctx, tx, "horizon", int64(n)) if err != nil { return 0, err } return uint64(start + 1), nil } type nodeUpdate struct { Ind int ID uint64 graphlog.NodeUpdate } func (qs *QuadStore) incNodesCnt(ctx context.Context, tx kv.Tx, deltas, newDeltas []nodeUpdate) ([]int, error) { var buf [binary.MaxVarintLen64]byte // increment nodes keys := make([]kv.Key, 0, len(deltas)) for _, d := range deltas { keys = append(keys, bucketKeyForHashRefs(d.Hash)) } sizes, err := tx.GetBatch(ctx, keys) if err != nil { return nil, err } var del []int for i, d := range deltas { k := keys[i] var sz int64 if sizes[i] != nil { szu, _ := binary.Uvarint(sizes[i]) sz = int64(szu) sizes[i] = nil // cannot reuse buffer since it belongs to kv } sz += int64(d.RefInc) if sz <= 0 { if err := tx.Del(ctx, k); err != nil { return del, err } mNodesDel.Inc() del = append(del, i) continue } n := binary.PutUvarint(buf[:], uint64(sz)) val := append([]byte{}, buf[:n]...) if err := tx.Put(ctx, k, val); err != nil { return del, err } mNodesUpd.Inc() } // create new nodes for _, d := range newDeltas { n := binary.PutUvarint(buf[:], uint64(d.RefInc)) val := append([]byte{}, buf[:n]...) if err := tx.Put(ctx, bucketKeyForHashRefs(d.Hash), val); err != nil { return nil, err } mNodesNew.Inc() } return del, nil } type resolvedNode struct { ID uint64 New bool } func (qs *QuadStore) incNodes(ctx context.Context, tx kv.Tx, deltas []graphlog.NodeUpdate) (map[refs.ValueHash]resolvedNode, error) { var ( ins []nodeUpdate upd = make([]nodeUpdate, 0, len(deltas)) ids = make(map[refs.ValueHash]resolvedNode, len(deltas)) ) err := qs.resolveValDeltas(ctx, tx, deltas, func(i int, id uint64) { if id == 0 { // not exists, should create ins = append(ins, nodeUpdate{Ind: i, NodeUpdate: deltas[i]}) } else { // exists, should update upd = append(upd, nodeUpdate{Ind: i, ID: id, NodeUpdate: deltas[i]}) ids[deltas[i].Hash] = resolvedNode{ID: id} } }) if err != nil { return ids, err } if len(ins) != 0 { // preallocate IDs start, err := qs.genIDs(ctx, tx, len(ins)) if err != nil { return ids, err } // create and index new nodes for i, iv := range ins { id := start + uint64(i) node, err := createNodePrimitive(iv.Val) if err != nil { return ids, err } node.ID = id ids[iv.Hash] = resolvedNode{ID: id, New: true} if err := qs.indexNode(ctx, tx, node, iv.Val); err != nil { return ids, err } ins[i].ID = id } } _, err = qs.incNodesCnt(ctx, tx, upd, ins) return ids, err } func (qs *QuadStore) decNodes(ctx context.Context, tx kv.Tx, deltas []graphlog.NodeUpdate, nodes map[refs.ValueHash]uint64) error { upds := make([]nodeUpdate, 0, len(deltas)) for i, d := range deltas { id := nodes[d.Hash] if id == 0 || d.RefInc == 0 { continue } upds = append(upds, nodeUpdate{Ind: i, ID: id, NodeUpdate: d}) } del, err := qs.incNodesCnt(ctx, tx, upds, nil) if err != nil { return err } for _, i := range del { d := upds[i] key := bucketForVal(d.Hash[0], d.Hash[1]).AppendBytes(d.Hash[:]) if err = tx.Del(ctx, key); err != nil { return err } if iri, ok := d.Val.(quad.IRI); ok { qs.valueLRU.Del(string(iri)) } if err := qs.delLog(ctx, tx, d.ID); err != nil { return err } } return nil } func (qs *QuadStore) NewQuadWriter() (quad.WriteCloser, error) { return &quadWriter{qs: qs}, nil } type quadWriter struct { qs *QuadStore tx kv.Tx err error n int } func (w *quadWriter) WriteQuad(q quad.Quad) error { _, err := w.WriteQuads([]quad.Quad{q}) return err } func (w *quadWriter) flush(ctx context.Context) error { w.n = 0 if err := w.qs.flushMapBucket(ctx, w.tx); err != nil { w.err = err return err } if err := w.tx.Commit(ctx); err != nil { w.qs.writer.Unlock() w.tx = nil w.err = err return err } tx, err := w.qs.db.Tx(ctx, true) if err != nil { w.qs.writer.Unlock() w.err = err return err } w.tx = wrapTx(tx) return nil } func (w *quadWriter) WriteQuads(buf []quad.Quad) (int, error) { mApplyBatch.Observe(float64(len(buf))) defer prometheus.NewTimer(mApplySeconds).ObserveDuration() ctx := context.TODO() if w.tx == nil { w.qs.writer.Lock() tx, err := w.qs.db.Tx(ctx, true) if err != nil { w.qs.writer.Unlock() w.err = err return 0, err } w.tx = wrapTx(tx) } deltas := graphlog.InsertQuads(buf) if _, err := w.qs.applyAddDeltas(ctx, w.tx, nil, deltas, graph.IgnoreOpts{IgnoreDup: true}); err != nil { w.err = err return 0, err } w.n += len(buf) if w.n >= quad.DefaultBatch*20 { if err := w.flush(ctx); err != nil { return 0, err } } return len(buf), nil } func (w *quadWriter) Close() error { if w.tx == nil { return w.err } defer w.qs.writer.Unlock() if w.err != nil { _ = w.tx.Close() w.tx = nil return w.err } ctx := context.TODO() // flush quad indexes and commit err := w.qs.flushMapBucket(ctx, w.tx) if err != nil { _ = w.tx.Close() w.tx = nil return err } err = w.tx.Commit(ctx) w.tx = nil return err } func (qs *QuadStore) applyAddDeltas(ctx context.Context, tx kv.Tx, in []graph.Delta, deltas *graphlog.Deltas, ignoreOpts graph.IgnoreOpts) (map[refs.ValueHash]resolvedNode, error) { // first add all new nodes nodes, err := qs.incNodes(ctx, tx, deltas.IncNode) if err != nil { return nil, err } deltas.IncNode = nil // resolve and insert all new quads links := make([]*cproto.Primitive, 0, len(deltas.QuadAdd)) qadd := make(map[[4]uint64]struct{}, len(deltas.QuadAdd)) for _, q := range deltas.QuadAdd { var link cproto.Primitive mustBeNew := false var qkey [4]uint64 for i, dir := range quad.Directions { n, ok := nodes[q.Quad.Get(dir)] if !ok { continue } mustBeNew = mustBeNew || n.New link.SetDirection(dir, n.ID) qkey[i] = n.ID } if _, ok := qadd[qkey]; ok { continue } qadd[qkey] = struct{}{} if !mustBeNew { p, err := qs.hasPrimitive(ctx, tx, &link, false) if err != nil { return nil, err } if p != nil { if ignoreOpts.IgnoreDup { continue // already exists, no need to insert } err = graph.ErrQuadExists if len(in) != 0 { return nil, &graph.DeltaError{Delta: in[q.Ind], Err: err} } return nil, err } } links = append(links, &link) } qadd = nil deltas.QuadAdd = nil qstart, err := qs.genIDs(ctx, tx, len(links)) if err != nil { return nil, err } for i := range links { links[i].ID = qstart + uint64(i) links[i].Timestamp = time.Now().UnixNano() } if err := qs.indexLinks(ctx, tx, links); err != nil { return nil, err } return nodes, nil } func (qs *QuadStore) ApplyDeltas(in []graph.Delta, ignoreOpts graph.IgnoreOpts) error { mApplyBatch.Observe(float64(len(in))) defer prometheus.NewTimer(mApplySeconds).ObserveDuration() ctx := context.TODO() qs.writer.Lock() defer qs.writer.Unlock() tx, err := qs.db.Tx(ctx, true) if err != nil { return err } defer tx.Close() tx = wrapTx(tx) deltas := graphlog.SplitDeltas(in) if len(deltas.QuadDel) != 0 || len(deltas.DecNode) != 0 { qs.mapNodes = nil } nodes, err := qs.applyAddDeltas(ctx, tx, in, deltas, ignoreOpts) if err != nil { return err } if len(deltas.QuadDel) != 0 || len(deltas.DecNode) != 0 { links := make([]*cproto.Primitive, 0, len(deltas.QuadDel)) // resolve all nodes that will be removed dnodes := make(map[refs.ValueHash]uint64, len(deltas.DecNode)) if err := qs.resolveValDeltas(ctx, tx, deltas.DecNode, func(i int, id uint64) { dnodes[deltas.DecNode[i].Hash] = id }); err != nil { return err } // check for existence and delete quads fixNodes := make(map[refs.ValueHash]int) for _, q := range deltas.QuadDel { link := new(cproto.Primitive) exists := true // resolve values of all quad directions // if any of the direction does not exists, the quad does not exists as well for _, dir := range quad.Directions { h := q.Quad.Get(dir) n, ok := nodes[h] if !ok { var id uint64 id, ok = dnodes[h] n.ID = id } if !ok { exists = exists && !h.Valid() continue } link.SetDirection(dir, n.ID) } if exists { p, err := qs.hasPrimitive(ctx, tx, link, true) if err != nil { return err } else if p == nil || p.Deleted { exists = false } else { link = p } } if !exists { if !ignoreOpts.IgnoreMissing { return &graph.DeltaError{Delta: in[q.Ind], Err: graph.ErrQuadNotExist} } // revert counters for all directions of this quad for _, dir := range quad.Directions { if h := q.Quad.Get(dir); h.Valid() { fixNodes[h]++ } } continue } links = append(links, link) } deltas.QuadDel = nil if err := qs.markLinksDead(ctx, tx, links); err != nil { return err } links = nil nodes = nil // we decremented some nodes that has non-existent quads - let's fix this if len(fixNodes) != 0 { for i, n := range deltas.DecNode { if dn := fixNodes[n.Hash]; dn != 0 { deltas.DecNode[i].RefInc += dn } } } // finally decrement and remove nodes if err := qs.decNodes(ctx, tx, deltas.DecNode, dnodes); err != nil { return err } deltas = nil dnodes = nil } // flush quad indexes and commit err = qs.flushMapBucket(ctx, tx) if err != nil { return err } return tx.Commit(ctx) } func (qs *QuadStore) indexNode(ctx context.Context, tx kv.Tx, p *cproto.Primitive, val quad.Value) error { var err error if val == nil { val, err = pquads.UnmarshalValue(p.Value) if err != nil { return err } } hash := quad.HashOf(val) err = tx.Put(ctx, bucketForVal(hash[0], hash[1]).AppendBytes(hash), uint64toBytes(p.ID)) if err != nil { return err } if iri, ok := val.(quad.IRI); ok { qs.valueLRU.Put(string(iri), p.ID) } if qs.mapNodes != nil { qs.mapNodes.Add(hash) } return qs.addToLog(ctx, tx, p) } func (qs *QuadStore) indexLinks(ctx context.Context, tx kv.Tx, links []*cproto.Primitive) error { for _, p := range links { if err := qs.indexLink(ctx, tx, p); err != nil { return err } } return qs.incSize(ctx, tx, int64(len(links))) } func (qs *QuadStore) indexLink(ctx context.Context, tx kv.Tx, p *cproto.Primitive) error { var err error qs.indexes.RLock() all := qs.indexes.all qs.indexes.RUnlock() for _, ind := range all { err = qs.addToMapBucket(tx, ind.KeyFor(p), p.ID) if err != nil { return err } } qs.bloomAdd(p) err = qs.indexSchema(tx, p) if err != nil { return err } return qs.addToLog(ctx, tx, p) } func (qs *QuadStore) markAsDead(ctx context.Context, tx kv.Tx, p *cproto.Primitive) error { p.Deleted = true //TODO(barakmich): Add tombstone? qs.bloomRemove(p) return qs.addToLog(ctx, tx, p) } func (qs *QuadStore) delLog(ctx context.Context, tx kv.Tx, id uint64) error { return tx.Del(ctx, logIndex.Append(uint64KeyBytes(id))) } func (qs *QuadStore) markLinksDead(ctx context.Context, tx kv.Tx, links []*cproto.Primitive) error { for _, p := range links { if err := qs.markAsDead(ctx, tx, p); err != nil { return err } } return qs.incSize(ctx, tx, -int64(len(links))) } func (qs *QuadStore) getBucketIndexes(ctx context.Context, tx kv.Tx, keys []kv.Key) ([][]uint64, error) { vals, err := tx.GetBatch(ctx, keys) if err != nil { return nil, err } out := make([][]uint64, len(keys)) for i, v := range vals { if len(v) == 0 { continue } ind, err := decodeIndex(v) if err != nil { return out, err } out[i] = ind } return out, nil } func countIndex(b []byte) (int64, error) { var cnt int64 for len(b) > 0 { _, n := binary.Uvarint(b) if n == 0 { return 0, io.ErrUnexpectedEOF } else if n < 0 { return 0, errors.New("varint: overflow") } cnt++ b = b[n:] } return cnt, nil } func decodeIndex(b []byte) ([]uint64, error) { var out []uint64 for len(b) > 0 { v, n := binary.Uvarint(b) if n == 0 { return out, io.ErrUnexpectedEOF } else if n < 0 { return out, errors.New("varint: overflow") } out = append(out, v) b = b[n:] } return out, nil } func appendIndex(bytelist []byte, l []uint64) []byte { b := make([]byte, len(bytelist)+(binary.MaxVarintLen64*len(l))) copy(b[:len(bytelist)], bytelist) off := len(bytelist) for _, x := range l { n := binary.PutUvarint(b[off:], x) off += n } return b[:off] } func (qs *QuadStore) bestUnique() ([]QuadIndex, error) { qs.indexes.RLock() ind := qs.indexes.exists qs.indexes.RUnlock() if len(ind) != 0 { return ind, nil } qs.indexes.Lock() defer qs.indexes.Unlock() if len(qs.indexes.exists) != 0 { return qs.indexes.exists, nil } for _, in := range qs.indexes.all { if in.Unique { if clog.V(2) { clog.Infof("using unique index: %v", in.Dirs) } qs.indexes.exists = []QuadIndex{in} return qs.indexes.exists, nil } } // TODO: find best combination of indexes inds := qs.indexes.all if len(inds) == 0 { return nil, fmt.Errorf("no indexes defined") } if clog.V(2) { clog.Infof("using index intersection: %v", inds) } qs.indexes.exists = inds return qs.indexes.exists, nil } func hasDir(dirs []quad.Direction, d quad.Direction) bool { for _, d2 := range dirs { if d == d2 { return true } } return false } func (qs *QuadStore) bestIndexes(dirs []quad.Direction) []QuadIndex { qs.indexes.RLock() all := qs.indexes.all qs.indexes.RUnlock() var ( max int // more specific index is better best QuadIndex ) for _, ind := range all { if len(ind.Dirs) < len(dirs) { continue // TODO(dennwc): allow intersecting indexes } match := 0 for i, d := range ind.Dirs { if i >= len(dirs) || !hasDir(dirs, d) { break } match++ } if match == len(dirs) { // exact index match return []QuadIndex{ind} } if match > 0 && match > max { best = ind max = match } } if max == 0 { return nil } // TODO(dennwc): intersect with some other index return []QuadIndex{best} } func (qs *QuadStore) hasPrimitive(ctx context.Context, tx kv.Tx, p *cproto.Primitive, get bool) (*cproto.Primitive, error) { if !qs.testBloom(p) { mQuadsBloomHit.Inc() return nil, nil } mQuadsBloomMiss.Inc() inds, err := qs.bestUnique() if err != nil { return nil, err } unique := len(inds) != 0 && inds[0].Unique keys := make([]kv.Key, len(inds)) for i, in := range inds { keys[i] = in.KeyFor(p) } lists, err := qs.getBucketIndexes(ctx, tx, keys) if err != nil { return nil, err } var options []uint64 for len(lists) > 0 { if len(lists) == 1 { options = lists[0] break } a, b := lists[0], lists[1] lists = lists[1:] a = intersectSortedUint64(a, b) lists[0] = a } if !get && unique { return p, nil } for i := len(options) - 1; i >= 0; i-- { // TODO: batch prim, err := qs.getPrimitiveFromLog(ctx, tx, options[i]) if err != nil { return nil, err } if prim.Deleted { continue } if prim.IsSameLink(p) { return prim, nil } } return nil, nil } func intersectSortedUint64(a, b []uint64) []uint64 { var c []uint64 boff := 0 outer: for _, x := range a { for { if boff >= len(b) { break outer } if x > b[boff] { boff++ continue } if x < b[boff] { break } if x == b[boff] { c = append(c, x) boff++ break } } } return c } func (qs *QuadStore) addToMapBucket(tx kv.Tx, key kv.Key, value uint64) error { if len(key) != 2 { return fmt.Errorf("trying to add to map bucket with invalid key: %v", key) } b, k := key[0], key[1] if len(k) == 0 { return fmt.Errorf("trying to add to map bucket %s with key 0", b) } if qs.mapBucket == nil { qs.mapBucket = make(map[string]map[string][]uint64) } bucket := string(b) m, ok := qs.mapBucket[bucket] if !ok { m = make(map[string][]uint64) qs.mapBucket[bucket] = m } m[string(k)] = append(m[string(k)], value) mIndexWriteBufferEntries.WithLabelValues(bucket).Inc() return nil } func (qs *QuadStore) flushMapBucket(ctx context.Context, tx kv.Tx) error { bs := make([]string, 0, len(qs.mapBucket)) for k := range qs.mapBucket { bs = append(bs, k) } sort.Strings(bs) for _, bucket := range bs { m := qs.mapBucket[bucket] if len(m) == 0 { continue } bloom := qs.mapBloom[bucket] mIndexWriteBufferFlushBatch.WithLabelValues(bucket).Observe(float64(len(m))) entryBytes := mIndexEntrySizeBytes.WithLabelValues(bucket) b := kv.Key{[]byte(bucket)} var ( keys []kv.Key keysPut []kv.Key ) if qs.mapBloom == nil { keys = make([]kv.Key, 0, len(m)) } for k := range m { bk := []byte(k) if qs.mapBloom != nil && (bloom == nil || !bloom.Test(bk)) { keysPut = append(keysPut, b.AppendBytes(bk)) } else { keys = append(keys, b.AppendBytes(bk)) } } sort.Sort(kv.ByKey(keysPut)) sort.Sort(kv.ByKey(keys)) vals, err := tx.GetBatch(ctx, keys) if err != nil { return err } if qs.mapBloom != nil && bloom == nil { bloom = boom.NewBloomFilter(100*1000*1000, 0.05) qs.mapBloom[bucket] = bloom } for _, k := range keysPut { l := m[string(k[1])] err = tx.Put(ctx, k, appendIndex(nil, l)) if err != nil { return err } if bloom != nil { bloom.Add(k[1]) } } for i, k := range keys { l := m[string(k[1])] buf := appendIndex(vals[i], l) entryBytes.Observe(float64(len(buf))) err = tx.Put(ctx, k, buf) if err != nil { return err } if bloom != nil { bloom.Add(k[1]) } } mIndexWriteBufferEntries.WithLabelValues(bucket).Set(0) } qs.mapBucket = nil return nil } func (qs *QuadStore) indexSchema(tx kv.Tx, p *cproto.Primitive) error { return nil } func (qs *QuadStore) addToLog(ctx context.Context, tx kv.Tx, p *cproto.Primitive) error { buf, err := proto.Marshal(p) if err != nil { return err } if err := tx.Put(ctx, logIndex.Append(uint64KeyBytes(p.ID)), buf); err != nil { return err } mPrimitiveAppend.Inc() return nil } func createNodePrimitive(v quad.Value) (*cproto.Primitive, error) { p := &cproto.Primitive{} b, err := pquads.MarshalValue(v) if err != nil { return p, err } p.Value = b p.Timestamp = time.Now().UnixNano() return p, nil } func (qs *QuadStore) resolveQuadValue(ctx context.Context, tx kv.Tx, v quad.Value) (uint64, error) { out, err := qs.resolveQuadValues(ctx, tx, []quad.Value{v}) if err != nil { return 0, err } return out[0], nil } func bucketKeyForVal(v quad.Value) kv.Key { hash := refs.HashOf(v) return bucketKeyForHash(hash) } func bucketKeyForHash(h refs.ValueHash) kv.Key { return bucketForVal(h[0], h[1]).AppendBytes(h[:]) } func bucketKeyForHashRefs(h refs.ValueHash) kv.Key { return bucketForValRefs(h[0], h[1]).AppendBytes(h[:]) } func (qs *QuadStore) resolveQuadValues(ctx context.Context, tx kv.Tx, vals []quad.Value) ([]uint64, error) { out := make([]uint64, len(vals)) inds := make([]int, 0, len(vals)) keys := make([]kv.Key, 0, len(vals)) for i, v := range vals { if iri, ok := v.(quad.IRI); ok { if x, ok := qs.valueLRU.Get(string(iri)); ok { out[i] = x.(uint64) continue } } else if v == nil { continue } inds = append(inds, i) keys = append(keys, bucketKeyForVal(v)) } if len(keys) == 0 { return out, nil } resp, err := tx.GetBatch(ctx, keys) if err != nil { return out, err } for i, b := range resp { if len(b) == 0 { continue } ind := inds[i] out[ind], _ = binary.Uvarint(b) if iri, ok := vals[ind].(quad.IRI); ok && out[ind] != 0 { qs.valueLRU.Put(string(iri), uint64(out[ind])) } } return out, nil } func uint64toBytes(x uint64) []byte { b := make([]byte, binary.MaxVarintLen64) return uint64toBytesAt(x, b) } func uint64toBytesAt(x uint64, bytes []byte) []byte { n := binary.PutUvarint(bytes, x) return bytes[:n] } func uint64KeyBytes(x uint64) kv.Key { k := make([]byte, 8) quadKeyEnc.PutUint64(k, x) return kv.Key{k} } func (qs *QuadStore) getPrimitivesFromLog(ctx context.Context, tx kv.Tx, keys []uint64) ([]*cproto.Primitive, error) { bkeys := make([]kv.Key, len(keys)) for i, k := range keys { bkeys[i] = logIndex.Append(uint64KeyBytes(k)) } vals, err := tx.GetBatch(ctx, bkeys) if err != nil { return nil, err } mPrimitiveFetch.Add(float64(len(vals))) out := make([]*cproto.Primitive, len(keys)) var last error for i, v := range vals { if v == nil { mPrimitiveFetchMiss.Inc() continue } var p cproto.Primitive if err = proto.Unmarshal(v, &p); err != nil { last = err } else { out[i] = &p } } return out, last } func (qs *QuadStore) getPrimitiveFromLog(ctx context.Context, tx kv.Tx, k uint64) (*cproto.Primitive, error) { out, err := qs.getPrimitivesFromLog(ctx, tx, []uint64{k}) if err != nil { return nil, err } else if out[0] == nil { return nil, kv.ErrNotFound } return out[0], nil } func (qs *QuadStore) initBloomFilter(ctx context.Context) error { if qs.exists.disabled { return nil } qs.exists.buf = make([]byte, 3*8) qs.exists.DeletableBloomFilter = boom.NewDeletableBloomFilter(100*1000*1000, 120, 0.05) return kv.View(ctx, qs.db, func(tx kv.Tx) error { p := cproto.Primitive{} it := tx.Scan(ctx, options.WithPrefixKV(logIndex)) defer it.Close() for it.Next(ctx) { v := it.Val() p = cproto.Primitive{} err := proto.Unmarshal(v, &p) if err != nil { return err } if p.IsNode() { continue } else if p.Deleted { continue } writePrimToBuf(&p, qs.exists.buf) qs.exists.Add(qs.exists.buf) } return it.Err() }) } func (qs *QuadStore) testBloom(p *cproto.Primitive) bool { if qs.exists.disabled { return true // false positives are expected } qs.exists.Lock() defer qs.exists.Unlock() writePrimToBuf(p, qs.exists.buf) return qs.exists.Test(qs.exists.buf) } func (qs *QuadStore) bloomRemove(p *cproto.Primitive) { if qs.exists.disabled { return } qs.exists.Lock() defer qs.exists.Unlock() writePrimToBuf(p, qs.exists.buf) qs.exists.TestAndRemove(qs.exists.buf) } func (qs *QuadStore) bloomAdd(p *cproto.Primitive) { if qs.exists.disabled { return } qs.exists.Lock() defer qs.exists.Unlock() writePrimToBuf(p, qs.exists.buf) qs.exists.Add(qs.exists.buf) } func writePrimToBuf(p *cproto.Primitive, buf []byte) { quadKeyEnc.PutUint64(buf[0:8], p.Subject) quadKeyEnc.PutUint64(buf[8:16], p.Predicate) quadKeyEnc.PutUint64(buf[16:24], p.Object) } type Int64Set []uint64 func (a Int64Set) Len() int { return len(a) } func (a Int64Set) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a Int64Set) Less(i, j int) bool { return a[i] < a[j] } ================================================ FILE: graph/kv/indexing_test.go ================================================ package kv import "testing" func TestIntersectSorted(t *testing.T) { tt := []struct { a []uint64 b []uint64 expect []uint64 }{ { a: []uint64{1, 2, 3, 4, 5, 6}, b: []uint64{2, 4, 6, 8, 10}, expect: []uint64{2, 4, 6}, }, { a: []uint64{6, 7, 8, 9, 10, 11}, b: []uint64{1, 2, 3, 4, 5, 6}, expect: []uint64{6}, }, } for i, x := range tt { c := intersectSortedUint64(x.a, x.b) if len(c) != len(x.expect) { t.Errorf("unexpected length: %d expected %d for test %d", len(c), len(x.expect), i) } for i, y := range c { if y != x.expect[i] { t.Errorf("unexpected entry: %#v expected %#v for test %d", c, x.expect, i) } } } } func TestIndexlist(t *testing.T) { init := []uint64{5, 10, 2340, 32432, 3243366} b := appendIndex(nil, init) out, err := decodeIndex(b) if err != nil { t.Fatalf("couldn't decodeIndex: %s", err) } if len(out) != len(init) { t.Fatalf("mismatched lengths. got %#v expected %#v", out, init) } for i := 0; i < len(out); i++ { if out[i] != init[i] { t.Fatalf("mismatched element %d. got %#v expected %#v", i, out, init) } } } ================================================ FILE: graph/kv/iterators.go ================================================ package kv import ( "context" "fmt" "github.com/hidal-go/hidalgo/kv" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query/shape" ) func (qs *QuadStore) NodesAllIterator() iterator.Shape { return qs.newAllIterator(true, nil) } func (qs *QuadStore) QuadsAllIterator() iterator.Shape { return qs.newAllIterator(false, nil) } func (qs *QuadStore) indexSize(ctx context.Context, ind QuadIndex, vals []uint64) (refs.Size, error) { var sz int64 err := kv.View(ctx, qs.db, func(tx kv.Tx) error { val, err := tx.Get(ctx, ind.Key(vals)) if err != nil { return err } sz, err = countIndex(val) if err != nil { return err } return nil }) if err != nil { return refs.Size{}, err } if len(ind.Dirs) == len(vals) { return refs.Size{ Value: sz, Exact: true, }, nil } return refs.Size{ Value: 1 + sz/2, Exact: false, }, nil } func (qs *QuadStore) QuadIteratorSize(ctx context.Context, d quad.Direction, v graph.Ref) (refs.Size, error) { vi, ok := v.(Int64Value) if !ok { return refs.Size{Value: 0, Exact: true}, nil } qs.indexes.RLock() all := qs.indexes.all qs.indexes.RUnlock() for _, ind := range all { if len(ind.Dirs) == 1 && ind.Dirs[0] == d { return qs.indexSize(ctx, ind, []uint64{uint64(vi)}) } } st, err := qs.Stats(ctx, false) if err != nil { return refs.Size{}, err } return refs.Size{ Value: 1 + st.Quads.Value/2, Exact: false, }, nil } func (qs *QuadStore) QuadIterator(dir quad.Direction, v graph.Ref) iterator.Shape { if v == nil { return iterator.NewNull() } vi, ok := v.(Int64Value) if !ok { return iterator.NewError(fmt.Errorf("unexpected node type: %T", v)) } // Find the best index for this direction. if ind := qs.bestIndexes([]quad.Direction{dir}); len(ind) == 1 { // this will scan the prefix automatically return qs.newQuadIterator(ind[0], []uint64{uint64(vi)}) } // Fallback: iterate all quads and check the corresponding direction. return qs.newAllIterator(false, &constraint{ dir: dir, val: vi, }) } func (qs *QuadStore) OptimizeShape(ctx context.Context, s shape.Shape) (shape.Shape, bool) { switch s := s.(type) { case shape.QuadsAction: return qs.optimizeQuadsAction(s) } return s, false } func (qs *QuadStore) optimizeQuadsAction(s shape.QuadsAction) (shape.Shape, bool) { if len(s.Filter) == 0 { return s, false } dirs := make([]quad.Direction, 0, len(s.Filter)) for d := range s.Filter { dirs = append(dirs, d) } ind := qs.bestIndexes(dirs) if len(ind) != 1 { return s, false // TODO(dennwc): allow intersecting indexes } quads := IndexScan{Index: ind[0]} for _, d := range ind[0].Dirs { v, ok := s.Filter[d].(Int64Value) if !ok { return s, false } quads.Values = append(quads.Values, uint64(v)) } return s.SimplifyFrom(quads), true } type IndexScan struct { Index QuadIndex Values []uint64 } func (s IndexScan) BuildIterator(qs graph.QuadStore) iterator.Shape { kqs, ok := qs.(*QuadStore) if !ok { return iterator.NewError(fmt.Errorf("expected KV quadstore, got: %T", qs)) } return kqs.newQuadIterator(s.Index, s.Values) } func (s IndexScan) Optimize(ctx context.Context, r shape.Optimizer) (shape.Shape, bool) { return s, false } ================================================ FILE: graph/kv/kvtest/kvtest.go ================================================ package kvtest import ( "context" "reflect" "testing" "github.com/cayleygraph/quad" hkv "github.com/hidal-go/hidalgo/kv" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest" "github.com/cayleygraph/cayley/graph/graphtest/testutil" "github.com/cayleygraph/cayley/graph/kv" "github.com/cayleygraph/cayley/query/shape" ) type DatabaseFunc func(t testing.TB) (hkv.KV, graph.Options, func()) type Config struct { AlwaysRunIntegration bool } func (c Config) quadStore() *graphtest.Config { return &graphtest.Config{ NoPrimitives: true, AlwaysRunIntegration: c.AlwaysRunIntegration, } } func newQuadStoreFunc(gen DatabaseFunc, bloom bool) testutil.DatabaseFunc { return func(t testing.TB) (graph.QuadStore, graph.Options) { return newQuadStore(t, gen, bloom) } } func NewQuadStoreFunc(gen DatabaseFunc) testutil.DatabaseFunc { return newQuadStoreFunc(gen, true) } func newQuadStore(t testing.TB, gen DatabaseFunc, bloom bool) (graph.QuadStore, graph.Options) { db, opt, closer := gen(t) if opt == nil { opt = make(graph.Options) } if !bloom { opt[kv.OptNoBloom] = true } err := kv.Init(db, opt) if err != nil { db.Close() closer() require.Fail(t, "init failed", "%v", err) } kdb, err := kv.New(db, opt) if err != nil { db.Close() closer() require.Fail(t, "create failed", "%v", err) } t.Cleanup(func() { kdb.Close() }) return kdb, opt } func NewQuadStore(t testing.TB, gen DatabaseFunc) (graph.QuadStore, graph.Options) { return newQuadStore(t, gen, true) } func TestAll(t *testing.T, gen DatabaseFunc, conf *Config) { if conf == nil { conf = &Config{} } qsgen := NewQuadStoreFunc(gen) t.Run("qs", func(t *testing.T) { graphtest.TestAll(t, qsgen, conf.quadStore()) }) qsgenNoBloom := newQuadStoreFunc(gen, false) t.Run("qs-no-bloom", func(t *testing.T) { graphtest.TestAll(t, qsgenNoBloom, conf.quadStore()) }) t.Run("optimize", func(t *testing.T) { testOptimize(t, gen, conf) }) } func testOptimize(t *testing.T, gen DatabaseFunc, _ *Config) { ctx := context.TODO() qs, opts := NewQuadStore(t, gen) testutil.MakeWriter(t, qs, opts, graphtest.MakeQuadSet()...) // With an linksto-fixed pair lto := shape.BuildIterator(ctx, qs, shape.Quads{ {Dir: quad.Object, Values: shape.Lookup{quad.Raw("F")}}, }) oldIt := shape.BuildIterator(ctx, qs, shape.Quads{ {Dir: quad.Object, Values: shape.Lookup{quad.Raw("F")}}, }).Iterate() defer oldIt.Close() newIts, ok := lto.Optimize(ctx) if ok { t.Errorf("unexpected optimization step") } if _, ok := newIts.(*kv.QuadIterator); !ok { t.Errorf("Optimized iterator type does not match original, got: %T", newIts) } newIt := newIts.Iterate() defer newIt.Close() newQuads := graphtest.IteratedQuadsNext(t, qs, newIt) oldQuads := graphtest.IteratedQuadsNext(t, qs, oldIt) if !reflect.DeepEqual(newQuads, oldQuads) { t.Errorf("Optimized iteration does not match original") } oldIt.Next(ctx) oldResults := make(map[string]graph.Ref) oldIt.TagResults(oldResults) newIt.Next(ctx) newResults := make(map[string]graph.Ref) newIt.TagResults(newResults) if !reflect.DeepEqual(newResults, oldResults) { t.Errorf("Discordant tag results, new:%v old:%v", newResults, oldResults) } } func BenchmarkAll(t *testing.B, gen DatabaseFunc, conf *Config) { if conf == nil { conf = &Config{} } qsgen := NewQuadStoreFunc(gen) t.Run("qs", func(t *testing.B) { graphtest.BenchmarkAll(t, qsgen, conf.quadStore()) }) } ================================================ FILE: graph/kv/leveldb/leveldb.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leveldb import ( "os" "github.com/cayleygraph/cayley/graph" hkv "github.com/hidal-go/hidalgo/kv" "github.com/hidal-go/hidalgo/kv/flat" "github.com/hidal-go/hidalgo/kv/flat/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" ) const ( Type = leveldb.Name ) func Create(path string, m graph.Options) (hkv.KV, error) { err := os.MkdirAll(path, 0700) if err != nil { return nil, err } db, err := leveldb.Open(path, &opt.Options{ ErrorIfExist: true, }) if os.IsExist(err) { return nil, graph.ErrDatabaseExists } else if err != nil { return nil, err } nosync, _ := m.BoolKey("nosync", false) db.SetWriteOptions(&opt.WriteOptions{ Sync: !nosync, }) return flat.Upgrade(db), nil } func Open(path string, m graph.Options) (hkv.KV, error) { db, err := leveldb.Open(path, &opt.Options{ ErrorIfMissing: true, }) if err != nil { return nil, err } nosync, _ := m.BoolKey("nosync", false) db.SetWriteOptions(&opt.WriteOptions{ Sync: !nosync, }) return flat.Upgrade(db), nil } ================================================ FILE: graph/kv/leveldb/leveldb_test.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leveldb import ( "io/ioutil" "os" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv/kvtest" hkv "github.com/hidal-go/hidalgo/kv" ) func makeLeveldb(t testing.TB) (hkv.KV, graph.Options, func()) { tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test_"+Type) if err != nil { t.Fatalf("Could not create working directory: %v", err) } db, err := Create(tmpDir, nil) if err != nil { os.RemoveAll(tmpDir) t.Fatal("Failed to create Bolt database.", err) } return db, nil, func() { db.Close() os.RemoveAll(tmpDir) } } func TestLeveldb(t *testing.T) { kvtest.TestAll(t, makeLeveldb, nil) } func BenchmarkLeveldb(b *testing.B) { kvtest.BenchmarkAll(b, makeLeveldb, nil) } ================================================ FILE: graph/kv/metrics.go ================================================ package kv import ( "context" "github.com/hidal-go/hidalgo/kv" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) var ( mApplyBatch = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "cayley_apply_deltas_batch", Help: "Number of quads in a buffer for ApplyDeltas or WriteQuads.", }) mApplySeconds = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "cayley_apply_deltas_seconds", Help: "Time to write a buffer in ApplyDeltas or WriteQuads.", }) mNodesNew = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_nodes_new_count", Help: "Number new nodes created.", }) mNodesUpd = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_nodes_upd_count", Help: "Number of node refcount updates.", }) mNodesDel = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_nodes_del_count", Help: "Number of node deleted.", }) mQuadsBloomHit = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_quads_bloom_hits", Help: "Number of times the quad bloom filter returned a negative result.", }) mQuadsBloomMiss = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_quads_bloom_miss", Help: "Number of times the quad bloom filter returned a positive result.", }) mPrimitiveFetch = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_primitive_fetch", Help: "Number of primitives fetched from KV.", }) mPrimitiveFetchMiss = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_primitive_fetch_miss", Help: "Number of primitives that were not found in KV.", }) mPrimitiveAppend = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_primitive_append", Help: "Number of primitives appended to log.", }) mIndexWriteBufferEntries = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "cayley_kv_index_buffer_entries", Help: "Number of entries in the index write buffer.", }, []string{"index"}) mIndexWriteBufferFlushBatch = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cayley_kv_index_buffer_flush_batch", Help: "Number of entries in the batch for flushing index entries.", }, []string{"index"}) mIndexEntrySizeBytes = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cayley_kv_index_entry_size_bytes", Help: "Size of a single index entry.", }, []string{"index"}) mKVGet = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_get_count", Help: "Number of get KV calls.", }) mKVGetMiss = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_get_miss", Help: "Number of get KV calls that found no value.", }) mKVGetSize = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "cayley_kv_get_size", Help: "Size of values returned from KV.", }) mKVPut = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_put_count", Help: "Number of put KV calls.", }) mKVPutSize = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "cayley_kv_put_size", Help: "Size of values put to KV.", }) mKVDel = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_del_count", Help: "Number of del KV calls.", }) mKVScan = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_scan_count", Help: "Number of scan KV calls.", }) mKVCommit = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_commit", Help: "Number of KV commits.", }) mKVCommitSeconds = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "cayley_kv_commit_seconds", Help: "Time to commit to KV.", }) mKVRollback = promauto.NewCounter(prometheus.CounterOpts{ Name: "cayley_kv_rollback", Help: "Number of KV rollbacks.", }) ) func wrapTx(tx kv.Tx) kv.Tx { return &mTx{tx: tx} } type mTx struct { tx kv.Tx done bool } func (tx *mTx) Commit(ctx context.Context) error { if !tx.done { tx.done = true mKVCommit.Inc() defer prometheus.NewTimer(mKVCommitSeconds).ObserveDuration() } return tx.tx.Commit(ctx) } func (tx *mTx) Close() error { if !tx.done { tx.done = true mKVRollback.Inc() } return tx.tx.Close() } func (tx *mTx) Get(ctx context.Context, key kv.Key) (kv.Value, error) { mKVGet.Inc() val, err := tx.tx.Get(ctx, key) if err == kv.ErrNotFound { mKVGetMiss.Inc() } else if err == nil { mKVGetSize.Observe(float64(len(val))) } return val, err } func (tx *mTx) GetBatch(ctx context.Context, keys []kv.Key) ([]kv.Value, error) { mKVGet.Add(float64(len(keys))) vals, err := tx.tx.GetBatch(ctx, keys) for _, v := range vals { if v == nil { mKVGetMiss.Inc() } else { mKVGetSize.Observe(float64(len(v))) } } return vals, err } func (tx *mTx) Put(ctx context.Context, k kv.Key, v kv.Value) error { mKVPut.Inc() mKVPutSize.Observe(float64(len(v))) return tx.tx.Put(ctx, k, v) } func (tx *mTx) Del(ctx context.Context, k kv.Key) error { mKVDel.Inc() return tx.tx.Del(ctx, k) } func (tx *mTx) Scan(ctx context.Context, opts ...kv.IteratorOption) kv.Iterator { mKVScan.Inc() return tx.tx.Scan(ctx, opts...) } ================================================ FILE: graph/kv/quad_iterator.go ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kv import ( "context" "fmt" "github.com/hidal-go/hidalgo/kv" "github.com/hidal-go/hidalgo/kv/options" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/proto" "github.com/cayleygraph/cayley/graph/refs" ) type QuadIterator struct { qs *QuadStore ind QuadIndex vals []uint64 size refs.Size err error } func (qs *QuadStore) newQuadIterator(ind QuadIndex, vals []uint64) *QuadIterator { return &QuadIterator{ qs: qs, ind: ind, vals: vals, size: refs.Size{Value: -1}, } } func (it *QuadIterator) Iterate() iterator.Scanner { return it.qs.newQuadIteratorNext(it.ind, it.vals) } func (it *QuadIterator) Lookup() iterator.Index { return it.qs.newQuadIteratorContains(it.ind, it.vals) } func (it *QuadIterator) SubIterators() []iterator.Shape { return nil } func (it *QuadIterator) getSize(ctx context.Context) (refs.Size, error) { if it.err != nil { return refs.Size{}, it.err } else if it.size.Value >= 0 { return it.size, nil } if len(it.ind.Dirs) == len(it.vals) { sz, err := it.qs.indexSize(ctx, it.ind, it.vals) if err != nil { it.err = err return refs.Size{}, it.err } it.size = sz return sz, nil } sz := refs.Size{Value: 1 + it.qs.Size()/2, Exact: false} it.size = sz return sz, nil } func (it *QuadIterator) String() string { return fmt.Sprintf("KVQuads(%v)", it.ind) } func (it *QuadIterator) Sorted() bool { return true } func (it *QuadIterator) Optimize(ctx context.Context) (iterator.Shape, bool) { return it, false } func (it *QuadIterator) Stats(ctx context.Context) (iterator.Costs, error) { s, err := it.getSize(ctx) return iterator.Costs{ ContainsCost: 1, NextCost: 2, Size: s, }, err } type quadIteratorNext struct { qs *QuadStore ind QuadIndex vals []uint64 tx kv.Tx it kv.Iterator done bool err error off int ids []uint64 buf []*proto.Primitive prim *proto.Primitive } func (qs *QuadStore) newQuadIteratorNext(ind QuadIndex, vals []uint64) *quadIteratorNext { return &quadIteratorNext{ qs: qs, ind: ind, vals: vals, } } func (it *quadIteratorNext) TagResults(dst map[string]graph.Ref) {} func (it *quadIteratorNext) Close() error { if it.it != nil { if err := it.it.Close(); err != nil && it.err == nil { it.err = err } if err := it.tx.Close(); err != nil && it.err == nil { it.err = err } it.it = nil it.tx = nil } return it.err } func (it *quadIteratorNext) Err() error { return it.err } func (it *quadIteratorNext) Result() graph.Ref { if it.off < 0 || it.prim == nil { return nil } return it.prim } func (it *quadIteratorNext) ensureTx(ctx context.Context) bool { if it.tx != nil { return true } it.tx, it.err = it.qs.db.Tx(ctx, false) if it.err != nil { return false } it.tx = wrapTx(it.tx) return true } func (it *quadIteratorNext) Next(ctx context.Context) bool { it.prim = nil if it.err != nil || it.done { return false } if it.it == nil { if !it.ensureTx(ctx) { return false } it.it = it.tx.Scan(ctx, options.WithPrefixKV(it.ind.Key(it.vals))) if err := it.Err(); err != nil { it.err = err return false } } for { if len(it.buf) == 0 { for len(it.ids[it.off:]) == 0 { it.off = 0 it.ids = nil it.buf = nil if !it.it.Next(ctx) { it.Close() it.done = true return false } it.ids, it.err = decodeIndex(it.it.Val()) if it.err != nil { return false } } ids := it.ids[it.off:] if len(ids) > nextBatch { ids = ids[:nextBatch] } it.buf, it.err = it.qs.getPrimitivesFromLog(ctx, it.tx, ids) if it.err != nil { return false } } else { it.buf, it.off = it.buf[1:], it.off+1 } for ; len(it.buf) > 0; it.buf, it.off = it.buf[1:], it.off+1 { p := it.buf[0] if p == nil || p.Deleted { continue } // TODO(dennwc): shouldn't this check the horizon? it.prim = p return true } } } func (it *quadIteratorNext) NextPath(ctx context.Context) bool { return false } func (it *quadIteratorNext) String() string { return fmt.Sprintf("KVQuadsNext(%v)", it.ind) } func (it *quadIteratorNext) Sorted() bool { return true } type quadIteratorContains struct { qs *QuadStore ind QuadIndex vals []uint64 err error prim *proto.Primitive } func (qs *QuadStore) newQuadIteratorContains(ind QuadIndex, vals []uint64) *quadIteratorContains { return &quadIteratorContains{ qs: qs, ind: ind, vals: vals, } } func (it *quadIteratorContains) TagResults(dst map[string]graph.Ref) {} func (it *quadIteratorContains) Close() error { return it.err } func (it *quadIteratorContains) Err() error { return it.err } func (it *quadIteratorContains) Result() graph.Ref { if it.prim == nil { return nil } return it.prim } func (it *quadIteratorContains) NextPath(ctx context.Context) bool { return false } func (it *quadIteratorContains) Contains(ctx context.Context, v graph.Ref) bool { it.prim = nil // TODO(dennwc): shouldn't this check the horizon? p, ok := v.(*proto.Primitive) if !ok { return false } for i, v := range it.vals { if p.GetDirection(it.ind.Dirs[i]) != v { return false } } it.prim = p return true } func (it *quadIteratorContains) String() string { return fmt.Sprintf("KVQuadsContains(%v)", it.ind) } func (it *quadIteratorContains) Sorted() bool { return true } ================================================ FILE: graph/kv/quadstore.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kv import ( "context" "encoding/binary" "encoding/json" "errors" "fmt" "os" "sync" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/pquads" "github.com/hidal-go/hidalgo/kv" boom "github.com/tylertreat/BoomFilters" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/proto" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/internal/lru" "github.com/cayleygraph/cayley/query/shape" ) var ( ErrNoBucket = errors.New("kv: no bucket") ErrEmptyPath = errors.New("kv: path to the database must be specified") ) type Registration struct { NewFunc NewFunc InitFunc InitFunc IsPersistent bool } type InitFunc func(string, graph.Options) (kv.KV, error) type NewFunc func(string, graph.Options) (kv.KV, error) func Register(name string, r Registration) { graph.RegisterQuadStore(name, graph.QuadStoreRegistration{ InitFunc: func(addr string, opt graph.Options) error { if !r.IsPersistent { return nil } kv, err := r.InitFunc(addr, opt) if err != nil { return err } defer kv.Close() if err = Init(kv, opt); err != nil { return err } return kv.Close() }, NewFunc: func(addr string, opt graph.Options) (graph.QuadStore, error) { kv, err := r.NewFunc(addr, opt) if err != nil { return nil, err } if !r.IsPersistent { if err = Init(kv, opt); err != nil { kv.Close() return nil, err } } return New(kv, opt) }, IsPersistent: r.IsPersistent, }) } const ( latestDataVersion = 2 envKVDefaultIndexes = "CAYLEY_KV_INDEXES" ) var ( _ refs.BatchNamer = (*QuadStore)(nil) _ shape.Optimizer = (*QuadStore)(nil) ) type QuadStore struct { db kv.KV indexes struct { sync.RWMutex all []QuadIndex // indexes used to detect duplicate quads exists []QuadIndex } valueLRU *lru.Cache writer sync.Mutex mapBucket map[string]map[string][]uint64 mapBloom map[string]*boom.BloomFilter mapNodes *boom.BloomFilter exists struct { disabled bool sync.Mutex buf []byte *boom.DeletableBloomFilter } } func newQuadStore(kv kv.KV) *QuadStore { return &QuadStore{db: kv} } func Init(kv kv.KV, opt graph.Options) error { ctx := context.TODO() qs := newQuadStore(kv) if data := os.Getenv(envKVDefaultIndexes); data != "" { qs.indexes.all = nil if err := json.Unmarshal([]byte(data), &qs.indexes); err != nil { return err } } if qs.indexes.all == nil { qs.indexes.all = DefaultQuadIndexes } if _, err := qs.getMetadata(ctx); err == nil { return graph.ErrDatabaseExists } else if err != ErrNoBucket { return err } upfront, err := opt.BoolKey("upfront", false) if err != nil { return err } if err := qs.createBuckets(ctx, upfront); err != nil { return err } if err := setVersion(ctx, qs.db, latestDataVersion); err != nil { return err } if err := qs.writeIndexesMeta(ctx); err != nil { return err } return nil } const ( OptNoBloom = "no_bloom" ) func New(kv kv.KV, opt graph.Options) (graph.QuadStore, error) { ctx := context.TODO() qs := newQuadStore(kv) if vers, err := qs.getMetadata(ctx); err == ErrNoBucket { return nil, graph.ErrNotInitialized } else if err != nil { return nil, err } else if vers != latestDataVersion { return nil, errors.New("kv: data version is out of date. Run cayleyupgrade for your config to update the data") } list, err := qs.readIndexesMeta(ctx) if err != nil { return nil, err } qs.indexes.all = list qs.valueLRU = lru.New(2000) qs.exists.disabled, _ = opt.BoolKey(OptNoBloom, false) if err := qs.initBloomFilter(ctx); err != nil { return nil, err } if !qs.exists.disabled { if sz, err := qs.getSize(); err != nil { return nil, err } else if sz == 0 { qs.mapBloom = make(map[string]*boom.BloomFilter) qs.mapNodes = boom.NewBloomFilter(100*1000*1000, 0.05) } } return qs, nil } func setVersion(ctx context.Context, db kv.KV, version int64) error { return kv.Update(ctx, db, func(tx kv.Tx) error { var buf [8]byte binary.LittleEndian.PutUint64(buf[:], uint64(version)) if err := tx.Put(ctx, metaBucket.AppendBytes([]byte("version")), buf[:]); err != nil { return fmt.Errorf("couldn't write version: %v", err) } return nil }) } func (qs *QuadStore) getMetaInt(ctx context.Context, key string) (int64, error) { var v int64 err := kv.View(ctx, qs.db, func(tx kv.Tx) error { val, err := tx.Get(ctx, metaBucket.AppendBytes([]byte(key))) if err == kv.ErrNotFound { return ErrNoBucket } else if err != nil { return err } v, err = asInt64(val, 0) if err != nil { return err } return nil }) return v, err } func (qs *QuadStore) getSize() (int64, error) { sz, err := qs.getMetaInt(context.TODO(), "size") if err == ErrNoBucket { return 0, nil } return sz, err } func (qs *QuadStore) Size() int64 { sz, _ := qs.getSize() return sz } func (qs *QuadStore) Stats(ctx context.Context, exact bool) (graph.Stats, error) { sz, err := qs.getMetaInt(ctx, "size") if err != nil { return graph.Stats{}, err } st := graph.Stats{ Nodes: refs.Size{ Value: sz / 3, Exact: false, // TODO(dennwc): store nodes count }, Quads: refs.Size{ Value: sz, Exact: true, }, } if exact { // calculate the exact number of nodes st.Nodes.Value = 0 it := qs.NodesAllIterator().Iterate() defer it.Close() for it.Next(ctx) { st.Nodes.Value++ } if err := it.Err(); err != nil { return st, err } st.Nodes.Exact = true } return st, nil } func (qs *QuadStore) Close() error { return qs.db.Close() } func (qs *QuadStore) getMetadata(ctx context.Context) (int64, error) { var vers int64 err := kv.View(ctx, qs.db, func(tx kv.Tx) error { val, err := tx.Get(ctx, metaBucket.AppendBytes([]byte("version"))) if err == kv.ErrNotFound { return ErrNoBucket } else if err != nil { return err } vers, err = asInt64(val, 0) if err != nil { return err } return nil }) return vers, err } func asInt64(b []byte, empty int64) (int64, error) { if len(b) == 0 { return empty, nil } else if len(b) != 8 { return 0, fmt.Errorf("unexpected int size: %d", len(b)) } v := int64(binary.LittleEndian.Uint64(b)) return v, nil } func (qs *QuadStore) horizon(ctx context.Context) int64 { h, _ := qs.getMetaInt(ctx, "horizon") return h } func (qs *QuadStore) ValuesOf(ctx context.Context, vals []graph.Ref) ([]quad.Value, error) { out := make([]quad.Value, len(vals)) var ( inds []int irefs []uint64 ) for i, v := range vals { if v == nil { continue } else if pv, ok := v.(refs.PreFetchedValue); ok { out[i] = pv.NameOf() continue } switch v := v.(type) { case Int64Value: if v == 0 { continue } inds = append(inds, i) irefs = append(irefs, uint64(v)) default: return out, fmt.Errorf("unknown type of graph.Ref; not meant for this quadstore. apparently a %#v", v) } } if len(irefs) == 0 { return out, nil } prim, err := qs.getPrimitives(ctx, irefs) if err != nil { return out, err } var last error for i, p := range prim { if p == nil || !p.IsNode() { continue } qv, err := pquads.UnmarshalValue(p.Value) if err != nil { last = err continue } out[inds[i]] = qv } return out, last } func (qs *QuadStore) RefsOf(ctx context.Context, nodes []quad.Value) ([]graph.Ref, error) { values := make([]graph.Ref, len(nodes)) err := kv.View(ctx, qs.db, func(tx kv.Tx) error { for i, node := range nodes { value, err := qs.resolveQuadValue(ctx, tx, node) if err != nil { return err } values[i] = Int64Value(value) } return nil }) if err != nil { return nil, err } return values, nil } func (qs *QuadStore) NameOf(v graph.Ref) (quad.Value, error) { ctx := context.TODO() vals, err := qs.ValuesOf(ctx, []graph.Ref{v}) if err != nil { return nil, fmt.Errorf("error getting NameOf %d: %w", v, err) } return vals[0], nil } func (qs *QuadStore) Quad(k graph.Ref) (quad.Quad, error) { key, ok := k.(*proto.Primitive) if !ok { return quad.Quad{}, fmt.Errorf("passed value was not a quad primitive: %T", k) } ctx := context.TODO() var v quad.Quad err := kv.View(ctx, qs.db, func(tx kv.Tx) error { var err error v, err = qs.primitiveToQuad(ctx, tx, key) return err }) if err == kv.ErrNotFound { err = nil } if err != nil { err = fmt.Errorf("error fetching quad %#v: %w", key, err) } return v, err } func (qs *QuadStore) primitiveToQuad(ctx context.Context, tx kv.Tx, p *proto.Primitive) (quad.Quad, error) { q := &quad.Quad{} for _, dir := range quad.Directions { v := p.GetDirection(dir) val, err := qs.getValFromLog(ctx, tx, v) if err != nil { return *q, err } q.Set(dir, val) } return *q, nil } func (qs *QuadStore) getValFromLog(ctx context.Context, tx kv.Tx, k uint64) (quad.Value, error) { if k == 0 { return nil, nil } p, err := qs.getPrimitiveFromLog(ctx, tx, k) if err != nil { return nil, err } return pquads.UnmarshalValue(p.Value) } func (qs *QuadStore) ValueOf(s quad.Value) (graph.Ref, error) { ctx := context.TODO() var out Int64Value err := kv.View(ctx, qs.db, func(tx kv.Tx) error { v, err := qs.resolveQuadValue(ctx, tx, s) out = Int64Value(v) return err }) if err != nil { return nil, err } if out == 0 { return nil, nil } return out, nil } func (qs *QuadStore) QuadDirection(val graph.Ref, d quad.Direction) (graph.Ref, error) { p, ok := val.(*proto.Primitive) if !ok { return nil, nil } switch d { case quad.Subject: return Int64Value(p.Subject), nil case quad.Predicate: return Int64Value(p.Predicate), nil case quad.Object: return Int64Value(p.Object), nil case quad.Label: if p.Label == 0 { return nil, nil } return Int64Value(p.Label), nil } return nil, nil } func (qs *QuadStore) getPrimitives(ctx context.Context, vals []uint64) ([]*proto.Primitive, error) { tx, err := qs.db.Tx(ctx, false) if err != nil { return nil, err } defer tx.Close() tx = wrapTx(tx) return qs.getPrimitivesFromLog(ctx, tx, vals) } type Int64Value uint64 func (v Int64Value) Key() interface{} { return v } ================================================ FILE: graph/kv/quadstore_test.go ================================================ package kv_test import ( "bytes" "context" "encoding/binary" henc "encoding/hex" "fmt" "sort" "sync" "testing" hkv "github.com/hidal-go/hidalgo/kv" "github.com/stretchr/testify/require" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/kv" "github.com/cayleygraph/cayley/graph/kv/btree" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/writer" ) func hex(s string) []byte { b, err := henc.DecodeString(s) if err != nil { panic(err) } return b } func irih(s string) []byte { h := refs.HashOf(quad.IRI(s)) return h[:] } func irib(s string) string { h := refs.HashOf(quad.IRI(s)) return string([]byte{'v', h[0], h[1]}) } func iric(s string) string { h := refs.HashOf(quad.IRI(s)) return string([]byte{'n', h[0], h[1]}) } func key(b string, k []byte) hkv.Key { return hkv.Key{[]byte(b), k} } func be(v ...uint64) []byte { b := make([]byte, 8*len(v)) for i, vi := range v { binary.BigEndian.PutUint64(b[i*8:], vi) } return b } func le(v uint64) []byte { var b [8]byte binary.LittleEndian.PutUint64(b[:], uint64(v)) return b[:] } const ( bMeta = "meta" bLog = "log" ) var ( kVers = []byte("version") vVers = le(2) vAuto = []byte("auto") kIndexes = []byte("indexes") ) type Ops []kvOp func (s Ops) Len() int { return len(s) } func (s Ops) Less(i, j int) bool { a, b := s[i], s[j] return a.key.Compare(b.key) < 0 } func (s Ops) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s Ops) String() string { buf := bytes.NewBuffer(nil) for _, op := range s { se := "" if op.err != nil { se = " (" + op.err.Error() + ")" } fmt.Fprintf(buf, "%v: %q = %x%s\n", op.typ, op.key, op.val, se) } return buf.String() } func TestApplyDeltas(t *testing.T) { kdb := btree.New() hook := &kvHook{db: kdb} expect := func(exp Ops) { got := hook.log() if len(exp) == len(got) { if false { sortByOp(exp, got) } // TODO: make node insert predictable for i, d := range exp { if bytes.Equal(d.key[0], vAuto) { exp[i].key = got[i].key } if bytes.Equal(d.val, vAuto) { exp[i].val = got[i].val } } } require.Equal(t, exp, got, "%d\n%v\nvs\n\n%d\n%v", len(exp), exp, len(got), got) } err := kv.Init(hook, nil) require.NoError(t, err) expect(Ops{ {opGet, key(bMeta, kVers), nil, hkv.ErrNotFound}, {opPut, key(bMeta, []byte{}), nil, nil}, {opPut, key(bLog, []byte{}), nil, nil}, {opPut, key("sp", []byte{}), nil, nil}, {opPut, key("ops", []byte{}), nil, nil}, {opPut, key(bMeta, kVers), vVers, nil}, {opPut, key(bMeta, kIndexes), []byte(`[{"dirs":"AQI=","unique":false},{"dirs":"AwIB","unique":false}]`), nil}, }) qs, err := kv.New(hook, nil) require.NoError(t, err) defer qs.Close() expect(Ops{ {opGet, key(bMeta, kVers), vVers, nil}, {opGet, key(bMeta, kIndexes), []byte(`[{"dirs":"AQI=","unique":false},{"dirs":"AwIB","unique":false}]`), nil}, {opGet, key(bMeta, []byte("size")), nil, hkv.ErrNotFound}, }) qw, err := writer.NewSingle(qs, graph.IgnoreOpts{}) require.NoError(t, err) err = qw.AddQuad(quad.MakeIRI("a", "b", "c", "")) require.NoError(t, err) expect(Ops{ {opGet, key(bMeta, []byte("horizon")), nil, hkv.ErrNotFound}, {opPut, key(bMeta, []byte("horizon")), le(3), nil}, {opPut, key(irib("a"), irih("a")), vAuto, nil}, {opPut, key(bLog, be(1)), vAuto, nil}, {opPut, key(irib("b"), irih("b")), vAuto, nil}, {opPut, key(bLog, be(2)), vAuto, nil}, {opPut, key(irib("c"), irih("c")), vAuto, nil}, {opPut, key(bLog, be(3)), vAuto, nil}, {opPut, key(iric("a"), irih("a")), hex("01"), nil}, {opPut, key(iric("b"), irih("b")), hex("01"), nil}, {opPut, key(iric("c"), irih("c")), hex("01"), nil}, {opGet, key(bMeta, []byte("horizon")), le(3), nil}, {opPut, key(bMeta, []byte("horizon")), le(4), nil}, {opPut, key(bLog, be(4)), vAuto, nil}, {opGet, key(bMeta, []byte("size")), nil, hkv.ErrNotFound}, {opPut, key(bMeta, []byte("size")), le(1), nil}, {opPut, key("ops", be(3, 2, 1)), hex("04"), nil}, {opPut, key("sp", be(1, 2)), hex("04"), nil}, }) err = qw.AddQuad(quad.MakeIRI("a", "b", "e", "")) require.NoError(t, err) expect(Ops{ // served from IRI cache //{opGet, irib("a"), irih("a"), vAuto, nil}, //{opGet, irib("b"), irih("b"), vAuto, nil}, {opGet, key(bMeta, []byte("horizon")), le(4), nil}, {opPut, key(bMeta, []byte("horizon")), le(5), nil}, {opPut, key(irib("e"), irih("e")), vAuto, nil}, {opPut, key(bLog, be(5)), vAuto, nil}, {opGet, key(iric("a"), irih("a")), hex("01"), nil}, {opGet, key(iric("b"), irih("b")), hex("01"), nil}, {opPut, key(iric("a"), irih("a")), hex("02"), nil}, {opPut, key(iric("b"), irih("b")), hex("02"), nil}, {opPut, key(iric("e"), irih("e")), hex("01"), nil}, {opGet, key(bMeta, []byte("horizon")), le(5), nil}, {opPut, key(bMeta, []byte("horizon")), le(6), nil}, {opPut, key(bLog, be(6)), vAuto, nil}, {opGet, key(bMeta, []byte("size")), le(1), nil}, {opPut, key(bMeta, []byte("size")), le(2), nil}, {opPut, key("ops", be(5, 2, 1)), hex("06"), nil}, {opGet, key("sp", be(1, 2)), hex("04"), nil}, {opPut, key("sp", be(1, 2)), hex("0406"), nil}, }) err = qw.RemoveQuad(quad.MakeIRI("a", "b", "c", "")) expect(Ops{ {opGet, key("sp", be(1, 2)), hex("0406"), nil}, {opGet, key("ops", be(3, 2, 1)), hex("04"), nil}, {opGet, key(bLog, be(4)), vAuto, nil}, {opPut, key(bLog, be(4)), vAuto, nil}, {opGet, key(bMeta, []byte("size")), le(2), nil}, {opPut, key(bMeta, []byte("size")), le(1), nil}, {opGet, key(iric("a"), irih("a")), hex("02"), nil}, {opGet, key(iric("b"), irih("b")), hex("02"), nil}, {opGet, key(iric("c"), irih("c")), hex("01"), nil}, {opPut, key(iric("a"), irih("a")), hex("01"), nil}, {opPut, key(iric("b"), irih("b")), hex("01"), nil}, {opDel, key(iric("c"), irih("c")), nil, nil}, {opDel, key(irib("c"), irih("c")), nil, nil}, {opDel, key(bLog, be(3)), nil, nil}, }) require.NoError(t, err) } func clone(b []byte) []byte { if b == nil { return nil } return append([]byte{}, b...) } func sortByOp(exp, got Ops) { // sort ops of one type li := -1 typ, b := -1, "" check := func(i int) { if li < 0 || i-li <= 0 { return } sort.Sort(exp[li:i]) sort.Sort(got[li:i]) //sort.Sort(bothOps{a: exp[li:i], b: got[li:i]}) li, typ, b = -1, -1, "" } for i, op := range exp { if op.typ != typ { check(i) } if li < 0 { li, typ, b = i, op.typ, string(op.key[0]) } } _ = b check(len(exp)) } const ( opGet = iota opPut opDel ) type kvOp struct { typ int key hkv.Key val hkv.Value err error } var _ hkv.KV = (*kvHook)(nil) type kvHook struct { db hkv.KV mu sync.Mutex ops Ops } func (h *kvHook) log() Ops { h.mu.Lock() ops := h.ops h.ops = nil h.mu.Unlock() return ops } func (h *kvHook) addOp(op kvOp) { h.mu.Lock() h.ops = append(h.ops, op) h.mu.Unlock() } func (h *kvHook) Close() error { return h.db.Close() } func (h *kvHook) Tx(ctx context.Context, rw bool) (hkv.Tx, error) { tx, err := h.db.Tx(ctx, rw) if err != nil { return nil, err } return txHook{h: h, tx: tx}, nil } func (h *kvHook) View(ctx context.Context, fn func(tx hkv.Tx) error) error { return h.db.View(ctx, func(tx hkv.Tx) error { return fn(txHook{h: h, tx: tx}) }) } func (h *kvHook) Update(ctx context.Context, fn func(tx hkv.Tx) error) error { return h.db.Update(ctx, func(tx hkv.Tx) error { return fn(txHook{h: h, tx: tx}) }) } type txHook struct { h *kvHook tx hkv.Tx } func (h txHook) Commit(ctx context.Context) error { return h.tx.Commit(ctx) } func (h txHook) Close() error { return h.tx.Close() } func (h txHook) GetBatch(ctx context.Context, keys []hkv.Key) ([]hkv.Value, error) { vals, err := h.tx.GetBatch(ctx, keys) if err != nil { return nil, err } for i, k := range keys { h.h.addOp(kvOp{ key: k.Clone(), val: vals[i].Clone(), }) } return vals, nil } func (h txHook) Get(ctx context.Context, k hkv.Key) (hkv.Value, error) { v, err := h.tx.Get(ctx, k) h.h.addOp(kvOp{ key: k.Clone(), val: v.Clone(), err: err, }) return v, err } func (h txHook) Put(ctx context.Context, k hkv.Key, v hkv.Value) error { err := h.tx.Put(ctx, k, v) h.h.addOp(kvOp{ typ: opPut, key: k.Clone(), val: v.Clone(), err: err, }) return err } func (h txHook) Del(ctx context.Context, k hkv.Key) error { err := h.tx.Del(ctx, k) h.h.addOp(kvOp{ typ: opDel, key: k.Clone(), err: err, }) return err } func (h txHook) Scan(ctx context.Context, opts ...hkv.IteratorOption) hkv.Iterator { return h.tx.Scan(ctx, opts...) } ================================================ FILE: graph/kv/registry.go ================================================ package kv import ( "strings" "github.com/hidal-go/hidalgo/kv" "github.com/cayleygraph/cayley/graph" ) func init() { for _, r := range kv.List() { switch r.Name { case "bolt", "bbolt": continue // legacy: register manually; see comments in the bolt package } r := r reg := Registration{ InitFunc: func(s string, options graph.Options) (kv.KV, error) { return r.OpenPath(s) }, NewFunc: func(s string, options graph.Options) (kv.KV, error) { return r.OpenPath(s) }, IsPersistent: !r.Volatile, } name := r.Name // override names for backward compatibility // names are also nicer without the "flat." prefix if strings.HasPrefix(name, "flat.") && !graph.IsRegistered(name[5:]) { name = name[5:] } Register(name, reg) } } ================================================ FILE: graph/linksto.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph // Defines one of the base iterators, the LinksTo iterator. A LinksTo takes a // subiterator of nodes, and contains an iteration of links which "link to" // those nodes in a given direction. // // Next()ing a LinksTo is straightforward -- iterate through all links to // // things in the subiterator, and then advance the subiterator, and do it again. // LinksTo is therefore sensitive to growing with a fanout. (A small-sized // subiterator could cause LinksTo to be large). // // Contains()ing a LinksTo means, given a link, take the direction we care about // and check if it's in our subiterator. Checking is therefore fairly cheap, and // similar to checking the subiterator alone. // // Can be seen as the dual of the HasA iterator. import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) // A LinksTo has a reference back to the graph.QuadStore (to create the iterators // for each node) the subiterator, and the direction the iterator comes from. // `next_it` is the tempoarary iterator held per result in `primary_it`. type LinksTo struct { qs QuadIndexer primary iterator.Shape dir quad.Direction size refs.Size } // NewLinksTo construct a new LinksTo iterator around a direction and a subiterator of // nodes. func NewLinksTo(qs QuadIndexer, it iterator.Shape, d quad.Direction) *LinksTo { return &LinksTo{ qs: qs, primary: it, dir: d, } } // Direction returns the direction under consideration. func (it *LinksTo) Direction() quad.Direction { return it.dir } func (it *LinksTo) Iterate() iterator.Scanner { return newLinksToNext(it.qs, it.primary.Iterate(), it.dir) } func (it *LinksTo) Lookup() iterator.Index { return newLinksToContains(it.qs, it.primary.Lookup(), it.dir) } func (it *LinksTo) String() string { return fmt.Sprintf("LinksTo(%v)", it.dir) } // SubIterators returns a list containing only our subiterator. func (it *LinksTo) SubIterators() []iterator.Shape { return []iterator.Shape{it.primary} } // Optimize the LinksTo, by replacing it if it can be. func (it *LinksTo) Optimize(ctx context.Context) (iterator.Shape, bool) { newPrimary, changed := it.primary.Optimize(ctx) if changed { it.primary = newPrimary if iterator.IsNull(it.primary) { return it.primary, true } } return it, false } // Stats returns a guess as to how big or costly it is to next the iterator. func (it *LinksTo) Stats(ctx context.Context) (iterator.Costs, error) { subitStats, err := it.primary.Stats(ctx) // TODO(barakmich): These should really come from the quadstore itself checkConstant := int64(1) nextConstant := int64(2) return iterator.Costs{ NextCost: nextConstant + subitStats.NextCost, ContainsCost: checkConstant + subitStats.ContainsCost, Size: it.getSize(ctx), }, err } func (it *LinksTo) getSize(ctx context.Context) refs.Size { if it.size.Value != 0 { return it.size } if fixed, ok := it.primary.(*iterator.Fixed); ok { // get real sizes from sub iterators var ( sz int64 exact = true ) for _, v := range fixed.Values() { sit := it.qs.QuadIterator(it.dir, v) st, _ := sit.Stats(ctx) sz += st.Size.Value exact = exact && st.Size.Exact } it.size.Value, it.size.Exact = sz, exact return it.size } stats, _ := it.qs.Stats(ctx, false) maxSize := stats.Quads.Value/2 + 1 // TODO(barakmich): It should really come from the quadstore itself const fanoutFactor = 20 st, _ := it.primary.Stats(ctx) value := st.Size.Value * fanoutFactor if value > maxSize { value = maxSize } it.size.Value, it.size.Exact = value, false return it.size } // A LinksTo has a reference back to the graph.QuadStore (to create the iterators // for each node) the subiterator, and the direction the iterator comes from. // `next_it` is the tempoarary iterator held per result in `primary_it`. type linksToNext struct { qs QuadIndexer primary iterator.Scanner dir quad.Direction nextIt iterator.Scanner result refs.Ref err error } // Construct a new LinksTo iterator around a direction and a subiterator of // nodes. func newLinksToNext(qs QuadIndexer, it iterator.Scanner, d quad.Direction) iterator.Scanner { return &linksToNext{ qs: qs, primary: it, dir: d, nextIt: iterator.NewNull().Iterate(), } } // Return the direction under consideration. func (it *linksToNext) Direction() quad.Direction { return it.dir } // Tag these results, and our subiterator's results. func (it *linksToNext) TagResults(dst map[string]refs.Ref) { it.primary.TagResults(dst) } func (it *linksToNext) String() string { return fmt.Sprintf("LinksToNext(%v)", it.dir) } // Next()ing a LinksTo operates as described above. func (it *linksToNext) Next(ctx context.Context) bool { for { if it.nextIt.Next(ctx) { it.result = it.nextIt.Result() return true } // If there's an error in the 'next' iterator, we save it and we're done. it.err = it.nextIt.Err() if it.err != nil { return false } // Subiterator is empty, get another one if !it.primary.Next(ctx) { // Possibly save error it.err = it.primary.Err() // We're out of nodes in our subiterator, so we're done as well. return false } it.nextIt.Close() it.nextIt = it.qs.QuadIterator(it.dir, it.primary.Result()).Iterate() // Continue -- return the first in the next set. } } func (it *linksToNext) Err() error { return it.err } func (it *linksToNext) Result() refs.Ref { return it.result } // Close closes the iterator. It closes all subiterators it can, but // returns the first error it encounters. func (it *linksToNext) Close() error { err := it.nextIt.Close() _err := it.primary.Close() if _err != nil && err == nil { err = _err } return err } // We won't ever have a new result, but our subiterators might. func (it *linksToNext) NextPath(ctx context.Context) bool { ok := it.primary.NextPath(ctx) if !ok { it.err = it.primary.Err() } return ok } // A LinksTo has a reference back to the graph.QuadStore (to create the iterators // for each node) the subiterator, and the direction the iterator comes from. // `next_it` is the tempoarary iterator held per result in `primary_it`. type linksToContains struct { qs QuadIndexer primary iterator.Index dir quad.Direction result refs.Ref err error } // Construct a new LinksTo iterator around a direction and a subiterator of // nodes. func newLinksToContains(qs QuadIndexer, it iterator.Index, d quad.Direction) iterator.Index { return &linksToContains{ qs: qs, primary: it, dir: d, } } // Return the direction under consideration. func (it *linksToContains) Direction() quad.Direction { return it.dir } // Tag these results, and our subiterator's results. func (it *linksToContains) TagResults(dst map[string]refs.Ref) { it.primary.TagResults(dst) } func (it *linksToContains) String() string { return fmt.Sprintf("LinksToContains(%v)", it.dir) } // If it checks in the right direction for the subiterator, it is a valid link // for the LinksTo. func (it *linksToContains) Contains(ctx context.Context, val refs.Ref) bool { node, err := it.qs.QuadDirection(val, it.dir) if err != nil { it.err = err return false } if it.primary.Contains(ctx, node) { it.result = val return true } return false } func (it *linksToContains) Err() error { if it.err != nil { return it.err } return it.primary.Err() } func (it *linksToContains) Result() refs.Ref { return it.result } // Close closes the iterator. It closes all subiterators it can, but // returns the first error it encounters. func (it *linksToContains) Close() error { return it.primary.Close() } // We won't ever have a new result, but our subiterators might. func (it *linksToContains) NextPath(ctx context.Context) bool { return it.primary.NextPath(ctx) } ================================================ FILE: graph/linksto_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph_test import ( "context" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphmock" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/quad" ) func TestLinksTo(t *testing.T) { ctx := context.TODO() object := quad.Raw("cool") q := quad.Quad{Subject: quad.IRI("alice"), Predicate: quad.IRI("is"), Object: object, Label: nil} qs := &graphmock.Store{ Data: []quad.Quad{q}, } fixed := iterator.NewFixed() val, err := qs.ValueOf(object) require.NoError(t, err) fixed.Add(val) lto := graph.NewLinksTo(qs, fixed, quad.Object).Iterate() require.True(t, lto.Next(ctx)) qv, err := qs.Quad(lto.Result()) require.NoError(t, err) require.Equal(t, q, qv) } ================================================ FILE: graph/log/graphlog.go ================================================ package graphlog import ( "bytes" "sort" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) type Op interface { isOp() } var ( _ Op = NodeUpdate{} _ Op = QuadUpdate{} ) type NodeUpdate struct { Hash refs.ValueHash Val quad.Value RefInc int } func (NodeUpdate) isOp() {} type QuadUpdate struct { Ind int Quad refs.QuadHash Del bool } func (QuadUpdate) isOp() {} type Deltas struct { IncNode []NodeUpdate DecNode []NodeUpdate QuadAdd []QuadUpdate QuadDel []QuadUpdate } func InsertQuads(in []quad.Quad) *Deltas { hnodes := make(map[refs.ValueHash]*NodeUpdate, len(in)*2) quadAdd := make([]QuadUpdate, 0, len(in)) for i, qd := range in { var q refs.QuadHash for _, dir := range quad.Directions { v := qd.Get(dir) if v == nil { continue } h := refs.HashOf(v) q.Set(dir, h) n := hnodes[h] if n == nil { n = &NodeUpdate{Hash: h, Val: v} hnodes[h] = n } n.RefInc++ } quadAdd = append(quadAdd, QuadUpdate{Ind: i, Quad: q}) } incNodes := make([]NodeUpdate, 0, len(hnodes)) for _, n := range hnodes { incNodes = append(incNodes, *n) } hnodes = nil sort.Slice(incNodes, func(i, j int) bool { return bytes.Compare(incNodes[i].Hash[:], incNodes[j].Hash[:]) < 0 }) return &Deltas{ IncNode: incNodes, QuadAdd: quadAdd, } } func SplitDeltas(in []graph.Delta) *Deltas { hnodes := make(map[refs.ValueHash]*NodeUpdate, len(in)*2) quadAdd := make([]QuadUpdate, 0, len(in)) quadDel := make([]QuadUpdate, 0, len(in)/2) var nadd, ndel int for i, d := range in { dn := 0 switch d.Action { case graph.Add: dn = +1 nadd++ case graph.Delete: dn = -1 ndel++ default: panic("unknown action") } var q refs.QuadHash for _, dir := range quad.Directions { v := d.Quad.Get(dir) if v == nil { continue } h := refs.HashOf(v) q.Set(dir, h) n := hnodes[h] if n == nil { n = &NodeUpdate{Hash: h, Val: v} hnodes[h] = n } n.RefInc += dn } u := QuadUpdate{Ind: i, Quad: q, Del: d.Action == graph.Delete} if !u.Del { quadAdd = append(quadAdd, u) } else { quadDel = append(quadDel, u) } } incNodes := make([]NodeUpdate, 0, nadd) decNodes := make([]NodeUpdate, 0, ndel) for _, n := range hnodes { if n.RefInc >= 0 { incNodes = append(incNodes, *n) } else { decNodes = append(decNodes, *n) } } sort.Slice(incNodes, func(i, j int) bool { return bytes.Compare(incNodes[i].Hash[:], incNodes[j].Hash[:]) < 0 }) sort.Slice(decNodes, func(i, j int) bool { return bytes.Compare(decNodes[i].Hash[:], decNodes[j].Hash[:]) < 0 }) hnodes = nil return &Deltas{ IncNode: incNodes, DecNode: decNodes, QuadAdd: quadAdd, QuadDel: quadDel, } } ================================================ FILE: graph/memstore/Makefile ================================================ # Copyright 2014 The Cayley Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. .PHONY: specify # Do not commit changes to this line unless you are satisfied # that the github.com/cznic/b tests AND the cayley integration # tests pass with the new sha. pinned=82d9e96a4503a42315b0fdf5201314302beafe06 specify: rm -rf b git clone https://github.com/cznic/b cd b && git checkout $(pinned) go test ./b @sed -e 's|interface{}[^{]*/\*K\*/|int64|g' -e 's|interface{}[^{]*/\*V\*/|\*primitive|g' b/btree.go >keys.go rm -rf b ================================================ FILE: graph/memstore/all_iterator.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package memstore import ( "context" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" ) var _ iterator.Shape = (*allIterator)(nil) type allIterator struct { qs *QuadStore all []*Primitive maxid int64 // id of last observed insert (prim id) nodes bool } func (qs *QuadStore) newAllIterator(nodes bool, maxid int64) *allIterator { return &allIterator{ qs: qs, all: qs.cloneAll(), nodes: nodes, maxid: maxid, } } func (it *allIterator) Iterate() iterator.Scanner { return it.qs.newAllIteratorNext(it.nodes, it.maxid, it.all) } func (it *allIterator) Lookup() iterator.Index { return it.qs.newAllIteratorContains(it.nodes, it.maxid) } func (it *allIterator) SubIterators() []iterator.Shape { return nil } func (it *allIterator) Optimize(ctx context.Context) (iterator.Shape, bool) { return it, false } func (it *allIterator) String() string { return "MemStoreAll" } func (it *allIterator) Stats(ctx context.Context) (iterator.Costs, error) { return iterator.Costs{ NextCost: 1, ContainsCost: 1, Size: refs.Size{ // TODO(dennwc): use maxid? Value: int64(len(it.all)), Exact: true, }, }, nil } func (p *Primitive) filter(isNode bool, maxid int64) bool { if p.ID > maxid { return false } else if isNode && p.Value != nil { return true } else if !isNode && !p.Quad.Zero() { return true } return false } type allIteratorNext struct { qs *QuadStore all []*Primitive maxid int64 // id of last observed insert (prim id) nodes bool i int // index into qs.all cur *Primitive done bool } func (qs *QuadStore) newAllIteratorNext(nodes bool, maxid int64, all []*Primitive) *allIteratorNext { return &allIteratorNext{ qs: qs, all: all, nodes: nodes, i: -1, maxid: maxid, } } func (it *allIteratorNext) ok(p *Primitive) bool { return p.filter(it.nodes, it.maxid) } func (it *allIteratorNext) Next(ctx context.Context) bool { it.cur = nil if it.done { return false } all := it.all if it.i >= len(all) { it.done = true return false } it.i++ for ; it.i < len(all); it.i++ { p := all[it.i] if p.ID > it.maxid { break } if it.ok(p) { it.cur = p return true } } it.done = true return false } func (it *allIteratorNext) Result() graph.Ref { if it.cur == nil { return nil } if !it.cur.Quad.Zero() { return qprim{p: it.cur} } return bnode(it.cur.ID) } func (it *allIteratorNext) Err() error { return nil } func (it *allIteratorNext) Close() error { it.done = true it.all = nil return nil } func (it *allIteratorNext) TagResults(dst map[string]graph.Ref) {} func (it *allIteratorNext) String() string { return "MemStoreAllNext" } func (it *allIteratorNext) NextPath(ctx context.Context) bool { return false } type allIteratorContains struct { qs *QuadStore maxid int64 // id of last observed insert (prim id) nodes bool cur *Primitive done bool } func (qs *QuadStore) newAllIteratorContains(nodes bool, maxid int64) *allIteratorContains { return &allIteratorContains{ qs: qs, nodes: nodes, maxid: maxid, } } func (it *allIteratorContains) ok(p *Primitive) bool { return p.filter(it.nodes, it.maxid) } func (it *allIteratorContains) Contains(ctx context.Context, v graph.Ref) bool { it.cur = nil if it.done { return false } id, ok := asID(v) if !ok { return false } p := it.qs.prim[id] if p.ID > it.maxid { return false } if !it.ok(p) { return false } it.cur = p return true } func (it *allIteratorContains) Result() graph.Ref { if it.cur == nil { return nil } if !it.cur.Quad.Zero() { return qprim{p: it.cur} } return bnode(it.cur.ID) } func (it *allIteratorContains) Err() error { return nil } func (it *allIteratorContains) Close() error { it.done = true return nil } func (it *allIteratorContains) TagResults(dst map[string]graph.Ref) {} func (it *allIteratorContains) String() string { return "MemStoreAllContains" } func (it *allIteratorContains) NextPath(ctx context.Context) bool { return false } ================================================ FILE: graph/memstore/gen.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:generate make specify package memstore ================================================ FILE: graph/memstore/iterator.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package memstore import ( "context" "fmt" "io" "math" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) type Iterator struct { qs *QuadStore tree *Tree d quad.Direction value int64 } func (qs *QuadStore) newIterator(tree *Tree, d quad.Direction, value int64) *Iterator { return &Iterator{ qs: qs, tree: tree, d: d, value: value, } } func (it *Iterator) Iterate() iterator.Scanner { // TODO(dennwc): it doesn't check the direction and value, while Contains does; is it expected? return it.qs.newIteratorNext(it.tree, it.d) } func (it *Iterator) Lookup() iterator.Index { return it.qs.newIteratorContains(it.tree, it.d, it.value) } func (it *Iterator) SubIterators() []iterator.Shape { return nil } func (it *Iterator) String() string { return fmt.Sprintf("MemStore(%v)", it.d) } func (it *Iterator) Sorted() bool { return true } func (it *Iterator) Optimize(ctx context.Context) (iterator.Shape, bool) { return it, false } func (it *Iterator) Stats(ctx context.Context) (iterator.Costs, error) { return iterator.Costs{ ContainsCost: int64(math.Log(float64(it.tree.Len()))) + 1, NextCost: 1, Size: refs.Size{ Value: int64(it.tree.Len()), Exact: true, }, }, nil } type iteratorNext struct { nodes bool qs *QuadStore tree *Tree d quad.Direction iter *Enumerator cur *Primitive err error } func (qs *QuadStore) newIteratorNext(tree *Tree, d quad.Direction) *iteratorNext { return &iteratorNext{ nodes: d == 0, d: d, qs: qs, tree: tree, } } func (it *iteratorNext) TagResults(dst map[string]graph.Ref) {} func (it *iteratorNext) Close() error { return nil } func (it *iteratorNext) Next(ctx context.Context) bool { if it.iter == nil { it.iter, it.err = it.tree.SeekFirst() if it.err == io.EOF || it.iter == nil { it.err = nil return false } else if it.err != nil { return false } } for { _, p, err := it.iter.Next() if err != nil { if err != io.EOF { it.err = err } return false } it.cur = p return true } } func (it *iteratorNext) Err() error { return it.err } func (it *iteratorNext) Result() graph.Ref { if it.cur == nil { return nil } return qprim{p: it.cur} } func (it *iteratorNext) NextPath(ctx context.Context) bool { return false } func (it *iteratorNext) String() string { return fmt.Sprintf("MemStoreNext(%v)", it.d) } func (it *iteratorNext) Sorted() bool { return true } type iteratorContains struct { nodes bool qs *QuadStore tree *Tree cur *Primitive d quad.Direction value int64 } func (qs *QuadStore) newIteratorContains(tree *Tree, d quad.Direction, value int64) *iteratorContains { return &iteratorContains{ nodes: d == 0, qs: qs, tree: tree, d: d, value: value, } } func (it *iteratorContains) TagResults(dst map[string]graph.Ref) {} func (it *iteratorContains) Close() error { return nil } func (it *iteratorContains) Err() error { return nil } func (it *iteratorContains) Result() graph.Ref { if it.cur == nil { return nil } return qprim{p: it.cur} } func (it *iteratorContains) NextPath(ctx context.Context) bool { return false } func (it *iteratorContains) Contains(ctx context.Context, v graph.Ref) bool { if v == nil { return false } switch v := v.(type) { case bnode: if p, ok := it.tree.Get(int64(v)); ok { it.cur = p return true } case qprim: if v.p.Quad.Dir(it.d) == it.value { it.cur = v.p return true } } return false } func (it *iteratorContains) String() string { return fmt.Sprintf("MemStoreContains(%v)", it.d) } func (it *iteratorContains) Sorted() bool { return true } ================================================ FILE: graph/memstore/keys.go ================================================ // Copyright 2014 The b Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package b implements a B+tree. // // Changelog // // 2014-06-26: Lower GC presure by recycling things. // // 2014-04-18: Added new method Put. // // Generic types // // Keys and their associated values are interface{} typed, similar to all of // the containers in the standard library. // // Semiautomatic production of a type specific variant of this package is // supported via // // $ make generic // // This command will write to stdout a version of the btree.go file where // every key type occurrence is replaced by the word 'key' (written in all // CAPS) and every value type occurrence is replaced by the word 'value' // (written in all CAPS). Then you have to replace these tokens with your // desired type(s), using any technique you're comfortable with. // // This is how, for example, 'example/int.go' was created: // // $ mkdir example // $ // $ # Note: the command bellow must be actually written using the words // $ # 'key' and 'value' in all CAPS. The proper form is avoided in this // $ # documentation to not confuse any text replacement mechanism. // $ // $ make generic | sed -e 's/key/int/g' -e 's/value/int/g' > example/int.go // // No other changes to int.go are necessary, it compiles just fine. // // Running the benchmarks for 1000 keys on a machine with Intel i5-4670 CPU @ // 3.4GHz, Go release 1.3. // // $ go test -bench 1e3 example/all_test.go example/int.go // PASS // BenchmarkSetSeq1e3 10000 146740 ns/op // BenchmarkGetSeq1e3 10000 108261 ns/op // BenchmarkSetRnd1e3 10000 254359 ns/op // BenchmarkGetRnd1e3 10000 134621 ns/op // BenchmarkDelRnd1e3 10000 211864 ns/op // BenchmarkSeekSeq1e3 10000 148628 ns/op // BenchmarkSeekRnd1e3 10000 215166 ns/op // BenchmarkNext1e3 200000 9211 ns/op // BenchmarkPrev1e3 200000 8843 ns/op // ok command-line-arguments 25.071s // $ package memstore import ( "fmt" "io" "sync" ) const ( kx = 32 //TODO benchmark tune this number if using custom key/value type(s). kd = 32 //TODO benchmark tune this number if using custom key/value type(s). ) func init() { if kd < 1 { panic(fmt.Errorf("kd %d: out of range", kd)) } if kx < 2 { panic(fmt.Errorf("kx %d: out of range", kx)) } } var ( btDPool = sync.Pool{New: func() interface{} { return &d{} }} btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}} btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}} btXPool = sync.Pool{New: func() interface{} { return &x{} }} ) type btTpool struct{ sync.Pool } func (p *btTpool) get(cmp Cmp) *Tree { x := p.Get().(*Tree) x.cmp = cmp return x } type btEpool struct{ sync.Pool } func (p *btEpool) get(err error, hit bool, i int, k int64, q *d, t *Tree, ver int64) *Enumerator { x := p.Get().(*Enumerator) x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver return x } type ( // Cmp compares a and b. Return value is: // // < 0 if a < b // 0 if a == b // > 0 if a > b // Cmp func(a, b int64) int d struct { // data page c int d [2*kd + 1]de n *d p *d } de struct { // d element k int64 v *Primitive } // Enumerator captures the state of enumerating a tree. It is returned // from the Seek* methods. The enumerator is aware of any mutations // made to the tree in the process of enumerating it and automatically // resumes the enumeration at the proper key, if possible. // // However, once an Enumerator returns io.EOF to signal "no more // items", it does no more attempt to "resync" on tree mutation(s). In // other words, io.EOF from an Enumaretor is "sticky" (idempotent). Enumerator struct { err error hit bool i int k int64 q *d t *Tree ver int64 } // Tree is a B+tree. Tree struct { c int cmp Cmp first *d last *d r interface{} ver int64 } xe struct { // x element ch interface{} k int64 } x struct { // index page c int x [2*kx + 2]xe } ) var ( // R/O zero values zd d zde de ze Enumerator zk int64 zt Tree zx x zxe xe ) func clr(q interface{}) { switch x := q.(type) { case *x: for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn clr(x.x[i].ch) } *x = zx btXPool.Put(x) case *d: *x = zd btDPool.Put(x) } } // -------------------------------------------------------------------------- x func newX(ch0 interface{}) *x { r := btXPool.Get().(*x) r.x[0].ch = ch0 return r } func (q *x) extract(i int) { q.c-- if i < q.c { copy(q.x[i:], q.x[i+1:q.c+1]) q.x[q.c].ch = q.x[q.c+1].ch q.x[q.c].k = zk // GC q.x[q.c+1] = zxe // GC } } func (q *x) insert(i int, k int64, ch interface{}) *x { c := q.c if i < c { q.x[c+1].ch = q.x[c].ch copy(q.x[i+2:], q.x[i+1:c]) q.x[i+1].k = q.x[i].k } c++ q.c = c q.x[i].k = k q.x[i+1].ch = ch return q } func (q *x) siblings(i int) (l, r *d) { if i >= 0 { if i > 0 { l = q.x[i-1].ch.(*d) } if i < q.c { r = q.x[i+1].ch.(*d) } } return } // -------------------------------------------------------------------------- d func (l *d) mvL(r *d, c int) { copy(l.d[l.c:], r.d[:c]) copy(r.d[:], r.d[c:r.c]) l.c += c r.c -= c } func (l *d) mvR(r *d, c int) { copy(r.d[c:], r.d[:r.c]) copy(r.d[:c], l.d[l.c-c:]) r.c += c l.c -= c } // ----------------------------------------------------------------------- Tree // TreeNew returns a newly created, empty Tree. The compare function is used // for key collation. func TreeNew(cmp Cmp) *Tree { return btTPool.get(cmp) } // Clear removes all K/V pairs from the tree. func (t *Tree) Clear() { if t.r == nil { return } clr(t.r) t.c, t.first, t.last, t.r = 0, nil, nil, nil t.ver++ } // Close performs Clear and recycles t to a pool for possible later reuse. No // references to t should exist or such references must not be used afterwards. func (t *Tree) Close() { t.Clear() *t = zt btTPool.Put(t) } func (t *Tree) cat(p *x, q, r *d, pi int) { t.ver++ q.mvL(r, r.c) if r.n != nil { r.n.p = q } else { t.last = q } q.n = r.n *r = zd btDPool.Put(r) if p.c > 1 { p.extract(pi) p.x[pi].ch = q } else { switch x := t.r.(type) { case *x: *x = zx btXPool.Put(x) case *d: *x = zd btDPool.Put(x) } t.r = q } } func (t *Tree) catX(p, q, r *x, pi int) { t.ver++ q.x[q.c].k = p.x[pi].k copy(q.x[q.c+1:], r.x[:r.c]) q.c += r.c + 1 q.x[q.c].ch = r.x[r.c].ch *r = zx btXPool.Put(r) if p.c > 1 { p.c-- pc := p.c if pi < pc { p.x[pi].k = p.x[pi+1].k copy(p.x[pi+1:], p.x[pi+2:pc+1]) p.x[pc].ch = p.x[pc+1].ch p.x[pc].k = zk // GC p.x[pc+1].ch = nil // GC } return } switch x := t.r.(type) { case *x: *x = zx btXPool.Put(x) case *d: *x = zd btDPool.Put(x) } t.r = q } // Delete removes the k's KV pair, if it exists, in which case Delete returns // true. func (t *Tree) Delete(k int64) (ok bool) { pi := -1 var p *x q := t.r if q == nil { return false } for { var i int i, ok = t.find(q, k) if ok { switch x := q.(type) { case *x: if x.c < kx && q != t.r { x, i = t.underflowX(p, x, pi, i) } pi = i + 1 p = x q = x.x[pi].ch ok = false continue case *d: t.extract(x, i) if x.c >= kd { return true } if q != t.r { t.underflow(p, x, pi) } else if t.c == 0 { t.Clear() } return true } } switch x := q.(type) { case *x: if x.c < kx && q != t.r { x, i = t.underflowX(p, x, pi, i) } pi = i p = x q = x.x[i].ch case *d: return false } } } func (t *Tree) extract(q *d, i int) { // (r *primitive) { t.ver++ //r = q.d[i].v // prepared for Extract q.c-- if i < q.c { copy(q.d[i:], q.d[i+1:q.c+1]) } q.d[q.c] = zde // GC t.c-- return } func (t *Tree) find(q interface{}, k int64) (i int, ok bool) { var mk int64 l := 0 switch x := q.(type) { case *x: h := x.c - 1 for l <= h { m := (l + h) >> 1 mk = x.x[m].k switch cmp := t.cmp(k, mk); { case cmp > 0: l = m + 1 case cmp == 0: return m, true default: h = m - 1 } } case *d: h := x.c - 1 for l <= h { m := (l + h) >> 1 mk = x.d[m].k switch cmp := t.cmp(k, mk); { case cmp > 0: l = m + 1 case cmp == 0: return m, true default: h = m - 1 } } } return l, false } // First returns the first item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. func (t *Tree) First() (k int64, v *Primitive) { if q := t.first; q != nil { q := &q.d[0] k, v = q.k, q.v } return } // Get returns the value associated with k and true if it exists. Otherwise Get // returns (zero-value, false). func (t *Tree) Get(k int64) (v *Primitive, ok bool) { q := t.r if q == nil { return } for { var i int if i, ok = t.find(q, k); ok { switch x := q.(type) { case *x: q = x.x[i+1].ch continue case *d: return x.d[i].v, true } } switch x := q.(type) { case *x: q = x.x[i].ch default: return } } } func (t *Tree) insert(q *d, i int, k int64, v *Primitive) *d { t.ver++ c := q.c if i < c { copy(q.d[i+1:], q.d[i:c]) } c++ q.c = c q.d[i].k, q.d[i].v = k, v t.c++ return q } // Last returns the last item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. func (t *Tree) Last() (k int64, v *Primitive) { if q := t.last; q != nil { q := &q.d[q.c-1] k, v = q.k, q.v } return } // Len returns the number of items in the tree. func (t *Tree) Len() int { return t.c } func (t *Tree) overflow(p *x, q *d, pi, i int, k int64, v *Primitive) { t.ver++ l, r := p.siblings(pi) if l != nil && l.c < 2*kd { l.mvL(q, 1) t.insert(q, i-1, k, v) p.x[pi-1].k = q.d[0].k return } if r != nil && r.c < 2*kd { if i < 2*kd { q.mvR(r, 1) t.insert(q, i, k, v) p.x[pi].k = r.d[0].k } else { t.insert(r, 0, k, v) p.x[pi].k = k } return } t.split(p, q, pi, i, k, v) } // Seek returns an Enumerator positioned on a an item such that k >= item's // key. ok reports if k == item.key The Enumerator's position is possibly // after the last item in the tree. func (t *Tree) Seek(k int64) (e *Enumerator, ok bool) { q := t.r if q == nil { e = btEPool.get(nil, false, 0, k, nil, t, t.ver) return } for { var i int if i, ok = t.find(q, k); ok { switch x := q.(type) { case *x: q = x.x[i+1].ch continue case *d: return btEPool.get(nil, ok, i, k, x, t, t.ver), true } } switch x := q.(type) { case *x: q = x.x[i].ch case *d: return btEPool.get(nil, ok, i, k, x, t, t.ver), false } } } // SeekFirst returns an enumerator positioned on the first KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. func (t *Tree) SeekFirst() (e *Enumerator, err error) { q := t.first if q == nil { return nil, io.EOF } return btEPool.get(nil, true, 0, q.d[0].k, q, t, t.ver), nil } // SeekLast returns an enumerator positioned on the last KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. func (t *Tree) SeekLast() (e *Enumerator, err error) { q := t.last if q == nil { return nil, io.EOF } return btEPool.get(nil, true, q.c-1, q.d[q.c-1].k, q, t, t.ver), nil } // Set sets the value associated with k. func (t *Tree) Set(k int64, v *Primitive) { //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) //defer func() { // dbg("--- POST\n%s\n====\n", t.dump()) //}() pi := -1 var p *x q := t.r if q == nil { z := t.insert(btDPool.Get().(*d), 0, k, v) t.r, t.first, t.last = z, z, z return } for { i, ok := t.find(q, k) if ok { switch x := q.(type) { case *x: if x.c > 2*kx { x, i = t.splitX(p, x, pi, i) } pi = i + 1 p = x q = x.x[i+1].ch continue case *d: x.d[i].v = v } return } switch x := q.(type) { case *x: if x.c > 2*kx { x, i = t.splitX(p, x, pi, i) } pi = i p = x q = x.x[i].ch case *d: switch { case x.c < 2*kd: t.insert(x, i, k, v) default: t.overflow(p, x, pi, i, k, v) } return } } } // Put combines Get and Set in a more efficient way where the tree is walked // only once. The upd(ater) receives (old-value, true) if a KV pair for k // exists or (zero-value, false) otherwise. It can then return a (new-value, // true) to create or overwrite the existing value in the KV pair, or // (whatever, false) if it decides not to create or not to update the value of // the KV pair. // // tree.Set(k, v) call conceptually equals calling // // tree.Put(k, func(int64, bool){ return v, true }) // // modulo the differing return values. func (t *Tree) Put(k int64, upd func(oldV *Primitive, exists bool) (newV *Primitive, write bool)) (oldV *Primitive, written bool) { pi := -1 var p *x q := t.r var newV *Primitive if q == nil { // new KV pair in empty tree newV, written = upd(newV, false) if !written { return } z := t.insert(btDPool.Get().(*d), 0, k, newV) t.r, t.first, t.last = z, z, z return } for { i, ok := t.find(q, k) if ok { switch x := q.(type) { case *x: if x.c > 2*kx { x, i = t.splitX(p, x, pi, i) } pi = i + 1 p = x q = x.x[i+1].ch continue case *d: oldV = x.d[i].v newV, written = upd(oldV, true) if !written { return } x.d[i].v = newV } return } switch x := q.(type) { case *x: if x.c > 2*kx { x, i = t.splitX(p, x, pi, i) } pi = i p = x q = x.x[i].ch case *d: // new KV pair newV, written = upd(newV, false) if !written { return } switch { case x.c < 2*kd: t.insert(x, i, k, newV) default: t.overflow(p, x, pi, i, k, newV) } return } } } func (t *Tree) split(p *x, q *d, pi, i int, k int64, v *Primitive) { t.ver++ r := btDPool.Get().(*d) if q.n != nil { r.n = q.n r.n.p = r } else { t.last = r } q.n = r r.p = q copy(r.d[:], q.d[kd:2*kd]) for i := range q.d[kd:] { q.d[kd+i] = zde } q.c = kd r.c = kd var done bool if i > kd { done = true t.insert(r, i-kd, k, v) } if pi >= 0 { p.insert(pi, r.d[0].k, r) } else { t.r = newX(q).insert(0, r.d[0].k, r) } if done { return } t.insert(q, i, k, v) } func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { t.ver++ r := btXPool.Get().(*x) copy(r.x[:], q.x[kx+1:]) q.c = kx r.c = kx if pi >= 0 { p.insert(pi, q.x[kx].k, r) q.x[kx].k = zk for i := range q.x[kx+1:] { q.x[kx+i+1] = zxe } switch { case i < kx: return q, i case i == kx: return p, pi default: // i > kx return r, i - kx - 1 } } nr := newX(q).insert(0, q.x[kx].k, r) t.r = nr q.x[kx].k = zk for i := range q.x[kx+1:] { q.x[kx+i+1] = zxe } switch { case i < kx: return q, i case i == kx: return nr, 0 default: // i > kx return r, i - kx - 1 } } func (t *Tree) underflow(p *x, q *d, pi int) { t.ver++ l, r := p.siblings(pi) if l != nil && l.c+q.c >= 2*kd { l.mvR(q, 1) p.x[pi-1].k = q.d[0].k } else if r != nil && q.c+r.c >= 2*kd { q.mvL(r, 1) p.x[pi].k = r.d[0].k r.d[r.c] = zde // GC } else if l != nil { t.cat(p, l, q, pi-1) } else { t.cat(p, q, r, pi) } } func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { t.ver++ var l, r *x if pi >= 0 { if pi > 0 { l = p.x[pi-1].ch.(*x) } if pi < p.c { r = p.x[pi+1].ch.(*x) } } if l != nil && l.c > kx { q.x[q.c+1].ch = q.x[q.c].ch copy(q.x[1:], q.x[:q.c]) q.x[0].ch = l.x[l.c].ch q.x[0].k = p.x[pi-1].k q.c++ i++ l.c-- p.x[pi-1].k = l.x[l.c].k return q, i } if r != nil && r.c > kx { q.x[q.c].k = p.x[pi].k q.c++ q.x[q.c].ch = r.x[0].ch p.x[pi].k = r.x[0].k copy(r.x[:], r.x[1:r.c]) r.c-- rc := r.c r.x[rc].ch = r.x[rc+1].ch r.x[rc].k = zk r.x[rc+1].ch = nil return q, i } if l != nil { i += l.c + 1 t.catX(p, l, q, pi-1) q = l return q, i } t.catX(p, q, r, pi) return q, i } // ----------------------------------------------------------------- Enumerator // Close recycles e to a pool for possible later reuse. No references to e // should exist or such references must not be used afterwards. func (e *Enumerator) Close() { *e = ze btEPool.Put(e) } // Next returns the currently enumerated item, if it exists and moves to the // next item in the key collation order. If there is no item to return, err == // io.EOF is returned. func (e *Enumerator) Next() (k int64, v *Primitive, err error) { if err = e.err; err != nil { return } if e.ver != e.t.ver { f, hit := e.t.Seek(e.k) if !e.hit && hit { if err = f.next(); err != nil { return } } *e = *f f.Close() } if e.q == nil { e.err, err = io.EOF, io.EOF return } if e.i >= e.q.c { if err = e.next(); err != nil { return } } i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, false e.next() return } func (e *Enumerator) next() error { if e.q == nil { e.err = io.EOF return io.EOF } switch { case e.i < e.q.c-1: e.i++ default: if e.q, e.i = e.q.n, 0; e.q == nil { e.err = io.EOF } } return e.err } // Prev returns the currently enumerated item, if it exists and moves to the // previous item in the key collation order. If there is no item to return, err // == io.EOF is returned. func (e *Enumerator) Prev() (k int64, v *Primitive, err error) { if err = e.err; err != nil { return } if e.ver != e.t.ver { f, hit := e.t.Seek(e.k) if !e.hit && hit { if err = f.prev(); err != nil { return } } *e = *f f.Close() } if e.q == nil { e.err, err = io.EOF, io.EOF return } if e.i >= e.q.c { if err = e.next(); err != nil { return } } i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, false e.prev() return } func (e *Enumerator) prev() error { if e.q == nil { e.err = io.EOF return io.EOF } switch { case e.i > 0: e.i-- default: if e.q = e.q.p; e.q == nil { e.err = io.EOF break } e.i = e.q.c - 1 } return e.err } ================================================ FILE: graph/memstore/keys_test.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package memstore import ( "math" "runtime/debug" "testing" "github.com/cznic/mathutil" ) func rng() *mathutil.FC32 { x, err := mathutil.NewFC32(math.MinInt32/4, math.MaxInt32/4, false) if err != nil { panic(err) } return x } func BenchmarkSetSeq1e3(b *testing.B) { benchmarkSetSeq(b, 1e3) } func BenchmarkSetSeq1e4(b *testing.B) { benchmarkSetSeq(b, 1e4) } func BenchmarkSetSeq1e5(b *testing.B) { benchmarkSetSeq(b, 1e5) } func BenchmarkSetSeq1e6(b *testing.B) { benchmarkSetSeq(b, 1e6) } func benchmarkSetSeq(b *testing.B, n int) { b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() r := TreeNew(cmp) debug.FreeOSMemory() b.StartTimer() for j := int64(0); j < int64(n); j++ { r.Set(j, nil) } b.StopTimer() r.Close() } b.StopTimer() } func BenchmarkGetSeq1e3(b *testing.B) { benchmarkGetSeq(b, 1e3) } func BenchmarkGetSeq1e4(b *testing.B) { benchmarkGetSeq(b, 1e4) } func BenchmarkGetSeq1e5(b *testing.B) { benchmarkGetSeq(b, 1e5) } func BenchmarkGetSeq1e6(b *testing.B) { benchmarkGetSeq(b, 1e6) } func benchmarkGetSeq(b *testing.B, n int) { r := TreeNew(cmp) for i := int64(0); i < int64(n); i++ { r.Set(i, nil) } debug.FreeOSMemory() b.ResetTimer() for i := 0; i < b.N; i++ { for j := int64(0); j < int64(n); j++ { r.Get(j) } } b.StopTimer() r.Close() } func BenchmarkSetRnd1e3(b *testing.B) { benchmarkSetRnd(b, 1e3) } func BenchmarkSetRnd1e4(b *testing.B) { benchmarkSetRnd(b, 1e4) } func BenchmarkSetRnd1e5(b *testing.B) { benchmarkSetRnd(b, 1e5) } func BenchmarkSetRnd1e6(b *testing.B) { benchmarkSetRnd(b, 1e6) } func benchmarkSetRnd(b *testing.B, n int) { rng := rng() a := make([]int, n) for i := range a { a[i] = rng.Next() } b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() r := TreeNew(cmp) debug.FreeOSMemory() b.StartTimer() for _, v := range a { r.Set(int64(v), nil) } b.StopTimer() r.Close() } b.StopTimer() } func BenchmarkGetRnd1e3(b *testing.B) { benchmarkGetRnd(b, 1e3) } func BenchmarkGetRnd1e4(b *testing.B) { benchmarkGetRnd(b, 1e4) } func BenchmarkGetRnd1e5(b *testing.B) { benchmarkGetRnd(b, 1e5) } func BenchmarkGetRnd1e6(b *testing.B) { benchmarkGetRnd(b, 1e6) } func benchmarkGetRnd(b *testing.B, n int) { r := TreeNew(cmp) rng := rng() a := make([]int64, n) for i := range a { a[i] = int64(rng.Next()) } for _, v := range a { r.Set(v, nil) } debug.FreeOSMemory() b.ResetTimer() for i := 0; i < b.N; i++ { for _, v := range a { r.Get(v) } } b.StopTimer() r.Close() } func BenchmarkDelSeq1e3(b *testing.B) { benchmarkDelSeq(b, 1e3) } func BenchmarkDelSeq1e4(b *testing.B) { benchmarkDelSeq(b, 1e4) } func BenchmarkDelSeq1e5(b *testing.B) { benchmarkDelSeq(b, 1e5) } func BenchmarkDelSeq1e6(b *testing.B) { benchmarkDelSeq(b, 1e6) } func benchmarkDelSeq(b *testing.B, n int) { b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() r := TreeNew(cmp) for j := int64(0); j < int64(n); j++ { r.Set(j, nil) } debug.FreeOSMemory() b.StartTimer() for j := int64(0); j < int64(n); j++ { r.Delete(j) } } b.StopTimer() } func BenchmarkDelRnd1e3(b *testing.B) { benchmarkDelRnd(b, 1e3) } func BenchmarkDelRnd1e4(b *testing.B) { benchmarkDelRnd(b, 1e4) } func BenchmarkDelRnd1e5(b *testing.B) { benchmarkDelRnd(b, 1e5) } func BenchmarkDelRnd1e6(b *testing.B) { benchmarkDelRnd(b, 1e6) } func benchmarkDelRnd(b *testing.B, n int) { rng := rng() a := make([]int64, n) for i := range a { a[i] = int64(rng.Next()) } b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() r := TreeNew(cmp) for _, v := range a { r.Set(v, nil) } debug.FreeOSMemory() b.StartTimer() for _, v := range a { r.Delete(v) } b.StopTimer() r.Close() } b.StopTimer() } func BenchmarkSeekSeq1e3(b *testing.B) { benchmarkSeekSeq(b, 1e3) } func BenchmarkSeekSeq1e4(b *testing.B) { benchmarkSeekSeq(b, 1e4) } func BenchmarkSeekSeq1e5(b *testing.B) { benchmarkSeekSeq(b, 1e5) } func BenchmarkSeekSeq1e6(b *testing.B) { benchmarkSeekSeq(b, 1e6) } func benchmarkSeekSeq(b *testing.B, n int) { for i := 0; i < b.N; i++ { b.StopTimer() t := TreeNew(cmp) for j := int64(0); j < int64(n); j++ { t.Set(j, nil) } debug.FreeOSMemory() b.StartTimer() for j := int64(0); j < int64(n); j++ { e, _ := t.Seek(j) e.Close() } b.StopTimer() t.Close() } b.StopTimer() } func BenchmarkSeekRnd1e3(b *testing.B) { benchmarkSeekRnd(b, 1e3) } func BenchmarkSeekRnd1e4(b *testing.B) { benchmarkSeekRnd(b, 1e4) } func BenchmarkSeekRnd1e5(b *testing.B) { benchmarkSeekRnd(b, 1e5) } func BenchmarkSeekRnd1e6(b *testing.B) { benchmarkSeekRnd(b, 1e6) } func benchmarkSeekRnd(b *testing.B, n int) { r := TreeNew(cmp) rng := rng() a := make([]int64, n) for i := range a { a[i] = int64(rng.Next()) } for _, v := range a { r.Set(v, nil) } debug.FreeOSMemory() b.ResetTimer() for i := 0; i < b.N; i++ { for _, v := range a { e, _ := r.Seek(v) e.Close() } } b.StopTimer() r.Close() } func BenchmarkNext1e3(b *testing.B) { benchmarkNext(b, 1e3) } func BenchmarkNext1e4(b *testing.B) { benchmarkNext(b, 1e4) } func BenchmarkNext1e5(b *testing.B) { benchmarkNext(b, 1e5) } func BenchmarkNext1e6(b *testing.B) { benchmarkNext(b, 1e6) } func benchmarkNext(b *testing.B, n int) { t := TreeNew(cmp) for i := int64(0); i < int64(n); i++ { t.Set(i, nil) } debug.FreeOSMemory() b.ResetTimer() for i := 0; i < b.N; i++ { en, err := t.SeekFirst() if err != nil { b.Fatal(err) } m := 0 for { if _, _, err = en.Next(); err != nil { break } m++ } if m != n { b.Fatal(m) } } b.StopTimer() t.Close() } func BenchmarkPrev1e3(b *testing.B) { benchmarkPrev(b, 1e3) } func BenchmarkPrev1e4(b *testing.B) { benchmarkPrev(b, 1e4) } func BenchmarkPrev1e5(b *testing.B) { benchmarkPrev(b, 1e5) } func BenchmarkPrev1e6(b *testing.B) { benchmarkPrev(b, 1e6) } func benchmarkPrev(b *testing.B, n int) { t := TreeNew(cmp) for i := int64(0); i < int64(n); i++ { t.Set(i, nil) } debug.FreeOSMemory() b.ResetTimer() for i := 0; i < b.N; i++ { en, err := t.SeekLast() if err != nil { b.Fatal(err) } m := 0 for { if _, _, err = en.Prev(); err != nil { break } m++ } if m != n { b.Fatal(m) } } } ================================================ FILE: graph/memstore/quadstore.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package memstore import ( "context" "fmt" "strconv" "strings" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) const QuadStoreType = "memstore" func init() { graph.RegisterQuadStore(QuadStoreType, graph.QuadStoreRegistration{ NewFunc: func(string, graph.Options) (graph.QuadStore, error) { return newQuadStore(), nil }, UpgradeFunc: nil, InitFunc: nil, IsPersistent: false, }) } type bnode int64 func (n bnode) Key() interface{} { return n } type qprim struct { p *Primitive } func (n qprim) Key() interface{} { return n.p.ID } var _ quad.Writer = (*QuadStore)(nil) func cmp(a, b int64) int { return int(a - b) } type QuadDirectionIndex struct { index [4]map[int64]*Tree } func NewQuadDirectionIndex() QuadDirectionIndex { return QuadDirectionIndex{[...]map[int64]*Tree{ quad.Subject - 1: make(map[int64]*Tree), quad.Predicate - 1: make(map[int64]*Tree), quad.Object - 1: make(map[int64]*Tree), quad.Label - 1: make(map[int64]*Tree), }} } func (qdi QuadDirectionIndex) Tree(d quad.Direction, id int64) *Tree { if d < quad.Subject || d > quad.Label { panic("illegal direction") } tree, ok := qdi.index[d-1][id] if !ok { tree = TreeNew(cmp) qdi.index[d-1][id] = tree } return tree } func (qdi QuadDirectionIndex) Get(d quad.Direction, id int64) (*Tree, bool) { if d < quad.Subject || d > quad.Label { panic("illegal direction") } tree, ok := qdi.index[d-1][id] return tree, ok } type Primitive struct { ID int64 Quad internalQuad Value quad.Value refs int } type internalQuad struct { S, P, O, L int64 } func (q internalQuad) Zero() bool { return q == (internalQuad{}) } func (q *internalQuad) SetDir(d quad.Direction, n int64) { switch d { case quad.Subject: q.S = n case quad.Predicate: q.P = n case quad.Object: q.O = n case quad.Label: q.L = n default: panic(fmt.Errorf("unknown dir: %v", d)) } } func (q internalQuad) Dir(d quad.Direction) int64 { var n int64 switch d { case quad.Subject: n = q.S case quad.Predicate: n = q.P case quad.Object: n = q.O case quad.Label: n = q.L } return n } type QuadStore struct { last int64 // TODO: string -> quad.Value once Raw -> typed resolution is unnecessary vals map[string]int64 quads map[internalQuad]int64 prim map[int64]*Primitive all []*Primitive // might not be sorted by id reading bool // someone else might be reading "all" slice - next insert/delete should clone it index QuadDirectionIndex horizon int64 // used only to assign ids to tx // vip_index map[string]map[int64]map[string]map[int64]*b.Tree } // New creates a new in-memory quad store and loads provided quads. func New(quads ...quad.Quad) *QuadStore { qs := newQuadStore() for _, q := range quads { qs.AddQuad(q) } return qs } func newQuadStore() *QuadStore { return &QuadStore{ vals: make(map[string]int64), quads: make(map[internalQuad]int64), prim: make(map[int64]*Primitive), index: NewQuadDirectionIndex(), } } func (qs *QuadStore) cloneAll() []*Primitive { qs.reading = true return qs.all } func (qs *QuadStore) addPrimitive(p *Primitive) int64 { qs.last++ id := qs.last p.ID = id p.refs = 1 qs.appendPrimitive(p) return id } func (qs *QuadStore) appendPrimitive(p *Primitive) { qs.prim[p.ID] = p if !qs.reading { qs.all = append(qs.all, p) } else { n := len(qs.all) qs.all = append(qs.all[:n:n], p) // reallocate slice qs.reading = false // this is a new slice } } const internalBNodePrefix = "memnode" func (qs *QuadStore) resolveVal(v quad.Value, add bool) (int64, bool) { if v == nil { return 0, false } if n, ok := v.(quad.BNode); ok && strings.HasPrefix(string(n), internalBNodePrefix) { n = n[len(internalBNodePrefix):] id, err := strconv.ParseInt(string(n), 10, 64) if err == nil && id != 0 { if p, ok := qs.prim[id]; ok || !add { if add { p.refs++ } return id, ok } qs.appendPrimitive(&Primitive{ID: id, refs: 1}) return id, true } } vs := v.String() if id, exists := qs.vals[vs]; exists || !add { if exists && add { qs.prim[id].refs++ } return id, exists } id := qs.addPrimitive(&Primitive{Value: v}) qs.vals[vs] = id return id, true } func (qs *QuadStore) resolveQuad(q quad.Quad, add bool) (internalQuad, bool) { var p internalQuad for dir := quad.Subject; dir <= quad.Label; dir++ { v := q.Get(dir) if v == nil { continue } if vid, _ := qs.resolveVal(v, add); vid != 0 { p.SetDir(dir, vid) } else if !add { return internalQuad{}, false } } return p, true } func (qs *QuadStore) lookupVal(id int64) quad.Value { pv := qs.prim[id] if pv == nil || pv.Value == nil { return quad.BNode(internalBNodePrefix + strconv.FormatInt(id, 10)) } return pv.Value } func (qs *QuadStore) lookupQuadDirs(p internalQuad) quad.Quad { var q quad.Quad for dir := quad.Subject; dir <= quad.Label; dir++ { vid := p.Dir(dir) if vid == 0 { continue } v := qs.lookupVal(vid) q.Set(dir, v) } return q } // AddNode adds a blank node (with no value) to quad store. It returns an id of the node. func (qs *QuadStore) AddBNode() int64 { return qs.addPrimitive(&Primitive{}) } // AddNode adds a value to quad store. It returns an id of the value. // False is returned as a second parameter if value exists already. func (qs *QuadStore) AddValue(v quad.Value) (int64, bool) { id, exists := qs.resolveVal(v, true) return id, !exists } func (qs *QuadStore) indexesForQuad(q internalQuad) []*Tree { trees := make([]*Tree, 0, 4) for dir := quad.Subject; dir <= quad.Label; dir++ { v := q.Dir(dir) if v == 0 { continue } trees = append(trees, qs.index.Tree(dir, v)) } return trees } // AddQuad adds a quad to quad store. It returns an id of the quad. // False is returned as a second parameter if quad exists already. func (qs *QuadStore) AddQuad(q quad.Quad) (int64, bool) { p, _ := qs.resolveQuad(q, false) if id := qs.quads[p]; id != 0 { return id, false } p, _ = qs.resolveQuad(q, true) pr := &Primitive{Quad: p} id := qs.addPrimitive(pr) qs.quads[p] = id for _, t := range qs.indexesForQuad(p) { t.Set(id, pr) } // TODO(barakmich): Add VIP indexing return id, true } // WriteQuad adds a quad to quad store. // // Deprecated: use AddQuad instead. func (qs *QuadStore) WriteQuad(q quad.Quad) error { qs.AddQuad(q) return nil } // WriteQuads implements quad.Writer. func (qs *QuadStore) WriteQuads(buf []quad.Quad) (int, error) { for _, q := range buf { qs.AddQuad(q) } return len(buf), nil } func (qs *QuadStore) NewQuadWriter() (quad.WriteCloser, error) { return &quadWriter{qs: qs}, nil } type quadWriter struct { qs *QuadStore } func (w *quadWriter) WriteQuad(q quad.Quad) error { w.qs.AddQuad(q) return nil } func (w *quadWriter) WriteQuads(buf []quad.Quad) (int, error) { for _, q := range buf { w.qs.AddQuad(q) } return len(buf), nil } func (w *quadWriter) Close() error { return nil } func (qs *QuadStore) deleteQuadNodes(q internalQuad) { for dir := quad.Subject; dir <= quad.Label; dir++ { id := q.Dir(dir) if id == 0 { continue } if p := qs.prim[id]; p != nil { p.refs-- if p.refs < 0 { panic("remove of deleted node") } else if p.refs == 0 { qs.Delete(id) } } } } func (qs *QuadStore) Delete(id int64) bool { p := qs.prim[id] if p == nil { return false } // remove from value index if p.Value != nil { delete(qs.vals, p.Value.String()) } // remove from quad indexes for _, t := range qs.indexesForQuad(p.Quad) { t.Delete(id) } delete(qs.quads, p.Quad) // remove primitive delete(qs.prim, id) di := -1 for i, p2 := range qs.all { if p == p2 { di = i break } } if di >= 0 { if !qs.reading { qs.all = append(qs.all[:di], qs.all[di+1:]...) } else { all := make([]*Primitive, 0, len(qs.all)-1) all = append(all, qs.all[:di]...) all = append(all, qs.all[di+1:]...) qs.all = all qs.reading = false // this is a new slice } } qs.deleteQuadNodes(p.Quad) return true } func (qs *QuadStore) findQuad(q quad.Quad) (int64, internalQuad, bool) { p, ok := qs.resolveQuad(q, false) if !ok { return 0, p, false } id := qs.quads[p] return id, p, id != 0 } func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta, ignoreOpts graph.IgnoreOpts) error { // Precheck the whole transaction (if required) if !ignoreOpts.IgnoreDup || !ignoreOpts.IgnoreMissing { for _, d := range deltas { switch d.Action { case graph.Add: if !ignoreOpts.IgnoreDup { if _, _, ok := qs.findQuad(d.Quad); ok { return &graph.DeltaError{Delta: d, Err: graph.ErrQuadExists} } } case graph.Delete: if !ignoreOpts.IgnoreMissing { if _, _, ok := qs.findQuad(d.Quad); !ok { return &graph.DeltaError{Delta: d, Err: graph.ErrQuadNotExist} } } default: return &graph.DeltaError{Delta: d, Err: graph.ErrInvalidAction} } } } for _, d := range deltas { switch d.Action { case graph.Add: qs.AddQuad(d.Quad) case graph.Delete: if id, _, ok := qs.findQuad(d.Quad); ok { qs.Delete(id) } default: // TODO: ideally we should rollback it return &graph.DeltaError{Delta: d, Err: graph.ErrInvalidAction} } } qs.horizon++ return nil } func asID(v graph.Ref) (int64, bool) { switch v := v.(type) { case bnode: return int64(v), true case qprim: return v.p.ID, true default: return 0, false } } func (qs *QuadStore) quad(v graph.Ref) (q internalQuad, ok bool) { switch v := v.(type) { case bnode: p := qs.prim[int64(v)] if p == nil { return } q = p.Quad case qprim: q = v.p.Quad default: return internalQuad{}, false } return q, !q.Zero() } func (qs *QuadStore) Quad(index graph.Ref) (quad.Quad, error) { q, ok := qs.quad(index) if !ok { return quad.Quad{}, nil } return qs.lookupQuadDirs(q), nil } func (qs *QuadStore) QuadIterator(d quad.Direction, value graph.Ref) iterator.Shape { id, ok := asID(value) if !ok { return iterator.NewNull() } index, ok := qs.index.Get(d, id) if ok && index.Len() != 0 { return qs.newIterator(index, d, id) } return iterator.NewNull() } func (qs *QuadStore) QuadIteratorSize(ctx context.Context, d quad.Direction, v graph.Ref) (refs.Size, error) { id, ok := asID(v) if !ok { return refs.Size{Value: 0, Exact: true}, nil } index, ok := qs.index.Get(d, id) if !ok { return refs.Size{Value: 0, Exact: true}, nil } return refs.Size{Value: int64(index.Len()), Exact: true}, nil } func (qs *QuadStore) Stats(ctx context.Context, exact bool) (graph.Stats, error) { return graph.Stats{ Nodes: refs.Size{ Value: int64(len(qs.vals)), Exact: true, }, Quads: refs.Size{ Value: int64(len(qs.quads)), Exact: true, }, }, nil } func (qs *QuadStore) ValueOf(name quad.Value) (graph.Ref, error) { if name == nil { return nil, nil } id := qs.vals[name.String()] if id == 0 { return nil, nil } return bnode(id), nil } func (qs *QuadStore) NameOf(v graph.Ref) (quad.Value, error) { if v == nil { return nil, nil } else if v, ok := v.(refs.PreFetchedValue); ok { return v.NameOf(), nil } n, ok := asID(v) if !ok { return nil, nil } if _, ok = qs.prim[n]; !ok { return nil, nil } return qs.lookupVal(n), nil } func (qs *QuadStore) QuadsAllIterator() iterator.Shape { return qs.newAllIterator(false, qs.last) } func (qs *QuadStore) QuadDirection(val graph.Ref, d quad.Direction) (graph.Ref, error) { q, ok := qs.quad(val) if !ok { return nil, nil } id := q.Dir(d) if id == 0 { return nil, nil } return bnode(id), nil } func (qs *QuadStore) NodesAllIterator() iterator.Shape { return qs.newAllIterator(true, qs.last) } func (qs *QuadStore) Close() error { return nil } ================================================ FILE: graph/memstore/quadstore_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package memstore import ( "context" "reflect" "sort" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/cayley/writer" ) // This is a simple test graph. // // +---+ +---+ // | A |------- ->| F |<-- // +---+ \------>+---+-/ +---+ \--+---+ // ------>|#B#| | | E | // +---+-------/ >+---+ | +---+ // | C | / v // +---+ -/ +---+ // ---- +---+/ |#G#| // \-->|#D#|------------->+---+ // +---+ var simpleGraph = []quad.Quad{ quad.MakeRaw("A", "follows", "B", ""), quad.MakeRaw("C", "follows", "B", ""), quad.MakeRaw("C", "follows", "D", ""), quad.MakeRaw("D", "follows", "B", ""), quad.MakeRaw("B", "follows", "F", ""), quad.MakeRaw("F", "follows", "G", ""), quad.MakeRaw("D", "follows", "G", ""), quad.MakeRaw("E", "follows", "F", ""), quad.MakeRaw("B", "status", "cool", "status_graph"), quad.MakeRaw("D", "status", "cool", "status_graph"), quad.MakeRaw("G", "status", "cool", "status_graph"), } func makeTestStore(data []quad.Quad) (*QuadStore, graph.QuadWriter, []pair) { seen := make(map[string]struct{}) qs := New() var ( val int64 ind []pair ) writer, _ := writer.NewSingleReplication(qs, nil) for _, t := range data { for _, dir := range quad.Directions { qp := t.GetString(dir) if _, ok := seen[qp]; !ok && qp != "" { val++ ind = append(ind, pair{qp, val}) seen[qp] = struct{}{} } } writer.AddQuad(t) val++ } return qs, writer, ind } func TestMemstore(t *testing.T) { graphtest.TestAll(t, func(t testing.TB) (graph.QuadStore, graph.Options) { return New(), nil }, &graphtest.Config{ AlwaysRunIntegration: true, }) } func BenchmarkMemstore(b *testing.B) { graphtest.BenchmarkAll(b, func(t testing.TB) (graph.QuadStore, graph.Options) { return New(), nil }, &graphtest.Config{ AlwaysRunIntegration: true, }) } type pair struct { query string value int64 } func TestMemstoreValueOf(t *testing.T) { qs, _, index := makeTestStore(simpleGraph) exp := graph.Stats{ Nodes: refs.Size{Value: 11, Exact: true}, Quads: refs.Size{Value: 11, Exact: true}, } st, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, exp, st, "Unexpected quadstore size") for _, test := range index { v, err := qs.ValueOf(quad.Raw(test.query)) require.NoError(t, err) switch v := v.(type) { default: t.Errorf("ValueOf(%q) returned unexpected type, got:%T expected int64", test.query, v) case bnode: require.Equal(t, test.value, int64(v)) } } } func TestIteratorsAndNextResultOrderA(t *testing.T) { ctx := context.TODO() qs, _, _ := makeTestStore(simpleGraph) fixed := iterator.NewFixed() qsv, err := qs.ValueOf(quad.Raw("C")) require.NoError(t, err) fixed.Add(qsv) fixed2 := iterator.NewFixed() qsv, err = qs.ValueOf(quad.Raw("follows")) require.NoError(t, err) fixed2.Add(qsv) all := qs.NodesAllIterator() const allTag = "all" innerAnd := iterator.NewAnd( graph.NewLinksTo(qs, fixed2, quad.Predicate), graph.NewLinksTo(qs, iterator.Tag(all, allTag), quad.Object), ) hasa := graph.NewHasA(qs, innerAnd, quad.Subject) outerAnd := iterator.NewAnd(fixed, hasa).Iterate() if !outerAnd.Next(ctx) { t.Error("Expected one matching subtree") } val := outerAnd.Result() vn, err := qs.NameOf(val) require.NoError(t, err) if vn != quad.Raw("C") { t.Errorf("Matching subtree should be %s, got %s", "barak", vn) } var ( got []string expect = []string{"B", "D"} ) for { m := make(map[string]graph.Ref, 1) outerAnd.TagResults(m) mv, err := qs.NameOf(m[allTag]) require.NoError(t, err) got = append(got, quad.ToString(mv)) if !outerAnd.NextPath(ctx) { break } } sort.Strings(got) if !reflect.DeepEqual(got, expect) { t.Errorf("Unexpected result, got:%q expect:%q", got, expect) } if outerAnd.Next(ctx) { t.Error("More than one possible top level output?") } } func TestLinksToOptimization(t *testing.T) { qs, _, _ := makeTestStore(simpleGraph) lto := shape.BuildIterator(context.TODO(), qs, shape.Quads{ {Dir: quad.Object, Values: shape.Lookup{quad.Raw("cool")}}, }) newIt, changed := lto.Optimize(context.TODO()) if changed { t.Errorf("unexpected optimization step") } if _, ok := newIt.(*Iterator); !ok { t.Fatal("Didn't swap out to LLRB") } } func TestRemoveQuad(t *testing.T) { ctx := context.TODO() qs, w, _ := makeTestStore(simpleGraph) err := w.RemoveQuad(quad.Make( "E", "follows", "F", nil, )) if err != nil { t.Error("Couldn't remove quad", err) } fixed := iterator.NewFixed() qsv, err := qs.ValueOf(quad.Raw("E")) require.NoError(t, err) fixed.Add(qsv) fixed2 := iterator.NewFixed() qsv, err = qs.ValueOf(quad.Raw("follows")) require.NoError(t, err) fixed2.Add(qsv) innerAnd := iterator.NewAnd( graph.NewLinksTo(qs, fixed, quad.Subject), graph.NewLinksTo(qs, fixed2, quad.Predicate), ) hasa := graph.NewHasA(qs, innerAnd, quad.Object) newIt, _ := hasa.Optimize(ctx) if newIt.Iterate().Next(ctx) { t.Error("E should not have any followers.") } } func TestTransaction(t *testing.T) { qs, w, _ := makeTestStore(simpleGraph) st, err := qs.Stats(context.Background(), true) require.NoError(t, err) tx := graph.NewTransaction() tx.AddQuad(quad.Make( "E", "follows", "G", nil)) tx.RemoveQuad(quad.Make( "Non", "existent", "quad", nil)) err = w.ApplyTransaction(tx) if err == nil { t.Error("Able to remove a non-existent quad") } st2, err := qs.Stats(context.Background(), true) require.NoError(t, err) require.Equal(t, st, st2, "Appended a new quad in a failed transaction") } ================================================ FILE: graph/nosql/all/all.go ================================================ package all import ( _ "github.com/hidal-go/hidalgo/legacy/nosql/all" _ "github.com/cayleygraph/cayley/graph/nosql" ) ================================================ FILE: graph/nosql/all/all_test.go ================================================ // +build docker package all import ( "testing" _ "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest/all" "github.com/cayleygraph/cayley/graph/nosql/nosqltest" hnosqltest "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" ) func TestNoSQL(t *testing.T) { hnosqltest.RunTest(t, nosqltest.TestAll) } func BenchmarkNoSQL(t *testing.B) { hnosqltest.RunBenchmark(t, nosqltest.BenchmarkAll) } ================================================ FILE: graph/nosql/elastic/elastic.go ================================================ package elastic import ( "context" "github.com/cayleygraph/cayley/graph" "github.com/hidal-go/hidalgo/legacy/nosql" "github.com/hidal-go/hidalgo/legacy/nosql/elastic" //import hidal-go first so the registration of the no sql stores occurs before quadstore iterates for registration gnosql "github.com/cayleygraph/cayley/graph/nosql" ) const Type = elastic.Name func Create(addr string, opt graph.Options) (nosql.Database, error) { return elastic.Dial(context.TODO(), addr, gnosql.DefaultDBName, nosql.Options(opt)) } func Open(addr string, opt graph.Options) (nosql.Database, error) { return elastic.Dial(context.TODO(), addr, gnosql.DefaultDBName, nosql.Options(opt)) } ================================================ FILE: graph/nosql/iterator.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package nosql import ( "context" "fmt" "github.com/hidal-go/hidalgo/legacy/nosql" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" ) type Linkage struct { Dir quad.Direction Val NodeHash } func linkageToFilters(links []Linkage) []nosql.FieldFilter { filters := make([]nosql.FieldFilter, 0, len(links)) for _, l := range links { filters = append(filters, nosql.FieldFilter{ Path: []string{l.Dir.String()}, Filter: nosql.Equal, Value: nosql.String(l.Val), }) } return filters } type Iterator struct { qs *QuadStore collection string limit int64 constraint []nosql.FieldFilter links []Linkage // used in Contains size refs.Size err error } func (qs *QuadStore) newLinksToIterator(collection string, links []Linkage) *Iterator { filters := linkageToFilters(links) it := qs.newIterator(collection, filters...) it.links = links return it } func (qs *QuadStore) newIterator(collection string, constraints ...nosql.FieldFilter) *Iterator { return &Iterator{ qs: qs, constraint: constraints, collection: collection, size: refs.Size{Value: -1}, } } func (it *Iterator) Iterate() iterator.Scanner { return it.qs.newIteratorNext(it.collection, it.constraint, it.limit) } func (it *Iterator) Lookup() iterator.Index { return it.qs.newIteratorContains(it.collection, it.constraint, it.links, it.limit) } func (it *Iterator) SubIterators() []iterator.Shape { return nil } func (it *Iterator) getSize(ctx context.Context) (refs.Size, error) { if it.size.Value == -1 { size, err := it.qs.getSize(it.collection, it.constraint) if err != nil { it.err = err } it.size = refs.Size{ Value: size, Exact: true, } } if it.limit > 0 && it.size.Value > it.limit { it.size.Value = it.limit } if it.size.Value < 0 { return refs.Size{ Value: it.qs.Size(), Exact: false, }, it.err } return it.size, nil } func (it *Iterator) Sorted() bool { return true } func (it *Iterator) Optimize(ctx context.Context) (iterator.Shape, bool) { return it, false } func (it *Iterator) String() string { return fmt.Sprintf("NoSQL(%v)", it.collection) } func (it *Iterator) Stats(ctx context.Context) (iterator.Costs, error) { size, err := it.getSize(ctx) return iterator.Costs{ ContainsCost: 1, NextCost: 5, Size: size, }, err } type iteratorNext struct { qs *QuadStore collection string limit int64 constraint []nosql.FieldFilter iter nosql.DocIterator result graph.Ref err error } func (qs *QuadStore) newIteratorNext(collection string, constraints []nosql.FieldFilter, limit int64) *iteratorNext { return &iteratorNext{ qs: qs, constraint: constraints, collection: collection, limit: limit, } } func (it *iteratorNext) makeIterator(ctx context.Context) nosql.DocIterator { q := it.qs.db.Query(it.collection) if len(it.constraint) != 0 { q = q.WithFields(it.constraint...) } if it.limit > 0 { q = q.Limit(int(it.limit)) } return q.Iterate(ctx) } func (it *iteratorNext) Close() error { if it.iter != nil { return it.iter.Close() } return nil } func (it *iteratorNext) TagResults(dst map[string]graph.Ref) {} func (it *iteratorNext) Next(ctx context.Context) bool { if it.iter == nil { it.iter = it.makeIterator(ctx) } var doc nosql.Document for { if !it.iter.Next(ctx) { if err := it.iter.Err(); err != nil { it.err = err clog.Errorf("error nexting iterator: %v", err) } return false } doc = it.iter.Doc() if it.collection == colQuads && !checkQuadValid(doc) { continue } break } if it.collection == colQuads { sh, _ := doc[fldSubject].(nosql.String) ph, _ := doc[fldPredicate].(nosql.String) oh, _ := doc[fldObject].(nosql.String) lh, _ := doc[fldLabel].(nosql.String) it.result = QuadHash{ string(sh), string(ph), string(oh), string(lh), } } else { id, _ := doc[fldHash].(nosql.String) it.result = NodeHash(id) } return true } func (it *iteratorNext) Err() error { return it.err } func (it *iteratorNext) Result() graph.Ref { return it.result } func (it *iteratorNext) NextPath(ctx context.Context) bool { return false } func (it *iteratorNext) Sorted() bool { return true } func (it *iteratorNext) String() string { return fmt.Sprintf("NoSQLNext(%v)", it.collection) } type iteratorContains struct { qs *QuadStore collection string limit int64 // FIXME(dennwc): doesn't work right now constraint []nosql.FieldFilter links []Linkage iter nosql.DocIterator result graph.Ref err error } func (qs *QuadStore) newIteratorContains(collection string, constraints []nosql.FieldFilter, links []Linkage, limit int64) *iteratorContains { return &iteratorContains{ qs: qs, collection: collection, constraint: constraints, links: links, limit: limit, } } func (it *iteratorContains) makeIterator(ctx context.Context) nosql.DocIterator { q := it.qs.db.Query(it.collection) if len(it.constraint) != 0 { q = q.WithFields(it.constraint...) } if it.limit > 0 { q = q.Limit(int(it.limit)) } return q.Iterate(ctx) } func (it *iteratorContains) Close() error { if it.iter != nil { return it.iter.Close() } return nil } func (it *iteratorContains) TagResults(dst map[string]graph.Ref) {} func (it *iteratorContains) Err() error { return it.err } func (it *iteratorContains) Result() graph.Ref { return it.result } func (it *iteratorContains) NextPath(ctx context.Context) bool { return false } func (it *iteratorContains) Contains(ctx context.Context, v graph.Ref) bool { if len(it.links) != 0 { qh := v.(QuadHash) for _, l := range it.links { if l.Val != NodeHash(qh.Get(l.Dir)) { return false } } it.result = v return true } if len(it.constraint) == 0 { it.result = v return true } qv, err := it.qs.NameOf(v) if err != nil { it.err = err return false } if qv == nil { return false } d := toDocumentValue(&it.qs.opt, qv) for _, f := range it.constraint { if !f.Matches(d) { return false } } it.result = v return true } func (it *iteratorContains) Sorted() bool { return true } func (it *iteratorContains) String() string { return fmt.Sprintf("NoSQLContains(%v)", it.collection) } ================================================ FILE: graph/nosql/mongo/mongo.go ================================================ package mongo import ( "context" "github.com/cayleygraph/cayley/graph" "github.com/hidal-go/hidalgo/legacy/nosql" "github.com/hidal-go/hidalgo/legacy/nosql/mongo" //import hidal-go first so the registration of the no sql stores occurs before quadstore iterates for registration gnosql "github.com/cayleygraph/cayley/graph/nosql" ) const Type = mongo.Name func Create(addr string, opt graph.Options) (nosql.Database, error) { return mongo.Dial(context.TODO(), addr, gnosql.DefaultDBName, nosql.Options(opt)) } func Open(addr string, opt graph.Options) (nosql.Database, error) { return mongo.Dial(context.TODO(), addr, gnosql.DefaultDBName, nosql.Options(opt)) } ================================================ FILE: graph/nosql/nosqltest/nosqltest.go ================================================ package nosqltest import ( "testing" "github.com/stretchr/testify/require" "github.com/hidal-go/hidalgo/legacy/nosql" "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest" gnosql "github.com/cayleygraph/cayley/graph/nosql" ) func toConfig(c nosql.Traits) graphtest.Config { return graphtest.Config{ NoPrimitives: true, TimeInMs: c.TimeInMs, OptimizesComparison: true, SkipDeletedFromIterator: true, SkipSizeCheckAfterDelete: true, } } func NewQuadStore(t testing.TB, gen nosqltest.Database) (graph.QuadStore, graph.Options) { db := gen.Run(t) err := gnosql.Init(db, nil) if err != nil { db.Close() require.Fail(t, "init failed", "%v", err) } tr := gen.Traits kdb, err := gnosql.NewQuadStore(db, &tr, nil) if err != nil { db.Close() require.Fail(t, "create failed", "%v", err) } t.Cleanup(func() { kdb.Close() }) return kdb, nil } func TestAll(t *testing.T, gen nosqltest.Database) { c := toConfig(gen.Traits) graphtest.TestAll(t, func(t testing.TB) (graph.QuadStore, graph.Options) { return NewQuadStore(t, gen) }, &c) } func BenchmarkAll(t *testing.B, gen nosqltest.Database) { c := toConfig(gen.Traits) graphtest.BenchmarkAll(t, func(t testing.TB) (graph.QuadStore, graph.Options) { return NewQuadStore(t, gen) }, &c) } ================================================ FILE: graph/nosql/ouch/ouch.go ================================================ package ouch import ( _ "github.com/hidal-go/hidalgo/legacy/nosql/couch" ) ================================================ FILE: graph/nosql/quadstore.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package nosql import ( "context" "encoding/base64" "fmt" "time" "github.com/hidal-go/hidalgo/legacy/nosql" "google.golang.org/protobuf/proto" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/pquads" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/internal/lru" ) const DefaultDBName = "cayley" type Registration struct { NewFunc NewFunc InitFunc InitFunc IsPersistent bool Traits } type Traits = nosql.Traits func init() { for _, reg := range nosql.List() { Register(reg.Name, Registration{ NewFunc: func(addr string, options graph.Options) (nosql.Database, error) { return reg.Open(context.TODO(), addr, DefaultDBName, nosql.Options(options)) }, InitFunc: func(addr string, options graph.Options) (nosql.Database, error) { return reg.New(context.TODO(), addr, DefaultDBName, nosql.Options(options)) }, IsPersistent: !reg.Volatile, Traits: reg.Traits, }) } } type InitFunc func(string, graph.Options) (nosql.Database, error) type NewFunc func(string, graph.Options) (nosql.Database, error) func Register(name string, r Registration) { graph.RegisterQuadStore(name, graph.QuadStoreRegistration{ InitFunc: func(addr string, opt graph.Options) error { if !r.IsPersistent { return nil } db, err := r.InitFunc(addr, opt) if err != nil { return err } defer db.Close() if err = Init(db, opt); err != nil { return err } return db.Close() }, NewFunc: func(addr string, opt graph.Options) (graph.QuadStore, error) { db, err := r.NewFunc(addr, opt) if err != nil { return nil, err } if !r.IsPersistent { if err = Init(db, opt); err != nil { db.Close() return nil, err } } nopt := r.Traits qs, err := NewQuadStore(db, &nopt, opt) if err != nil { return nil, err } return qs, nil }, IsPersistent: r.IsPersistent, }) } func Init(db nosql.Database, opt graph.Options) error { return ensureIndexes(context.TODO(), db) } func NewQuadStore(db nosql.Database, nopt *Traits, opt graph.Options) (*QuadStore, error) { if err := ensureIndexes(context.TODO(), db); err != nil { return nil, err } qs := &QuadStore{ db: db, ids: lru.New(1 << 16), sizes: lru.New(1 << 16), } if nopt != nil { qs.opt = *nopt } return qs, nil } type NodeHash string func (NodeHash) IsNode() bool { return false } func (v NodeHash) Key() interface{} { return v } func (v NodeHash) key() nosql.Key { return nosql.Key{string(v)} } type QuadHash [4]string func (QuadHash) IsNode() bool { return false } func (v QuadHash) Key() interface{} { return v } func (v QuadHash) Get(d quad.Direction) string { var ind int switch d { case quad.Subject: ind = 0 case quad.Predicate: ind = 1 case quad.Object: ind = 2 case quad.Label: ind = 3 } return v[ind] } const ( colLog = "log" colNodes = "nodes" colQuads = "quads" fldLogID = "id" fldSubject = "subject" fldPredicate = "predicate" fldObject = "object" fldLabel = "label" fldQuadAdded = "added" fldQuadDeleted = "deleted" fldHash = "hash" fldValue = "value" fldSize = "refs" fldValData = "str" fldIRI = "iri" fldBNode = "bnode" fldType = "type" fldLang = "lang" fldValInt = "int" fldValStrInt = "int_str" fldValFloat = "float" fldValBool = "bool" fldValTime = "ts" fldValPb = "pb" ) type QuadStore struct { db nosql.Database ids *lru.Cache sizes *lru.Cache opt Traits } func ensureIndexes(ctx context.Context, db nosql.Database) error { err := db.EnsureIndex(ctx, colLog, nosql.Index{ Fields: []string{fldLogID}, Type: nosql.StringExact, }, nil) if err != nil { return err } err = db.EnsureIndex(ctx, colNodes, nosql.Index{ Fields: []string{fldHash}, Type: nosql.StringExact, }, nil) if err != nil { return err } err = db.EnsureIndex(ctx, colQuads, nosql.Index{ Fields: []string{ fldSubject, fldPredicate, fldObject, fldLabel, }, Type: nosql.StringExact, }, []nosql.Index{ {Fields: []string{fldSubject}, Type: nosql.StringExact}, {Fields: []string{fldPredicate}, Type: nosql.StringExact}, {Fields: []string{fldObject}, Type: nosql.StringExact}, {Fields: []string{fldLabel}, Type: nosql.StringExact}, }) if err != nil { return err } return nil } func getKeyForQuad(t quad.Quad) nosql.Key { return nosql.Key{ hashOf(t.Subject), hashOf(t.Predicate), hashOf(t.Object), hashOf(t.Label), } } func hashOf(s quad.Value) string { if s == nil { return "" } h := quad.HashOf(s) return base64.StdEncoding.EncodeToString(h) } func (qs *QuadStore) nameToKey(name quad.Value) nosql.Key { node := qs.hashOf(name) return node.key() } func (qs *QuadStore) updateNodeBy(ctx context.Context, key nosql.Key, name quad.Value, inc int) error { if inc == 0 { return nil } d := toDocumentValue(&qs.opt, name) err := qs.db.Update(colNodes, key).Upsert(d).Inc(fldSize, inc).Do(ctx) if err != nil { return fmt.Errorf("error updating node: %v", err) } return nil } func (qs *QuadStore) cleanupNodes(ctx context.Context, keys []nosql.Key) error { err := qs.db.Delete(colNodes).Keys(keys...).WithFields(nosql.FieldFilter{ Path: []string{fldSize}, Filter: nosql.Equal, Value: nosql.Int(0), }).Do(ctx) if err != nil { err = fmt.Errorf("error cleaning up nodes: %v", err) } return err } func (qs *QuadStore) updateQuad(ctx context.Context, q quad.Quad, proc graph.Procedure) error { var setname string if proc == graph.Add { setname = fldQuadAdded } else if proc == graph.Delete { setname = fldQuadDeleted } doc := nosql.Document{ fldSubject: nosql.String(hashOf(q.Subject)), fldPredicate: nosql.String(hashOf(q.Predicate)), fldObject: nosql.String(hashOf(q.Object)), } if l := hashOf(q.Label); l != "" { doc[fldLabel] = nosql.String(l) } err := qs.db.Update(colQuads, getKeyForQuad(q)).Upsert(doc). Inc(setname, 1).Do(ctx) if err != nil { err = fmt.Errorf("quad update failed: %v", err) } return err } func checkQuadValid(q nosql.Document) bool { added, _ := asInt(q[fldQuadAdded]) deleted, _ := asInt(q[fldQuadDeleted]) return added > deleted } func (qs *QuadStore) checkValidQuad(ctx context.Context, key nosql.Key) (bool, error) { q, err := qs.db.FindByKey(ctx, colQuads, key) if err == nosql.ErrNotFound { return false, nil } if err != nil { err = fmt.Errorf("error checking quad validity: %v", err) return false, err } return checkQuadValid(q), nil } func (qs *QuadStore) batchInsert(col string) nosql.DocWriter { return nosql.BatchInsert(qs.db, col) } func (qs *QuadStore) appendLog(ctx context.Context, deltas []graph.Delta) ([]nosql.Key, error) { w := qs.batchInsert(colLog) defer w.Close() for _, d := range deltas { data, err := proto.Marshal(pquads.MakeQuad(d.Quad)) if err != nil { return w.Keys(), err } var action string if d.Action == graph.Add { action = "AddQuadPQ" } else { action = "DeleteQuadPQ" } err = w.WriteDoc(ctx, nil, nosql.Document{ "op": nosql.String(action), "data": nosql.Bytes(data), "ts": nosql.Time(time.Now().UTC()), }) if err != nil { return w.Keys(), err } } err := w.Flush(ctx) return w.Keys(), err } func (qs *QuadStore) NewQuadWriter() (quad.WriteCloser, error) { return &quadWriter{qs: qs}, nil } type quadWriter struct { qs *QuadStore deltas []graph.Delta } func (w *quadWriter) WriteQuad(q quad.Quad) error { _, err := w.WriteQuads([]quad.Quad{q}) return err } func (w *quadWriter) WriteQuads(buf []quad.Quad) (int, error) { // TODO(dennwc): write an optimized implementation w.deltas = w.deltas[:0] if cap(w.deltas) < len(buf) { w.deltas = make([]graph.Delta, 0, len(buf)) } for _, q := range buf { w.deltas = append(w.deltas, graph.Delta{ Quad: q, Action: graph.Add, }) } err := w.qs.ApplyDeltas(w.deltas, graph.IgnoreOpts{ IgnoreDup: true, }) w.deltas = w.deltas[:0] if err != nil { return 0, err } return len(buf), nil } func (w *quadWriter) Close() error { w.deltas = nil return nil } func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta, ignoreOpts graph.IgnoreOpts) error { ctx := context.TODO() ids := make(map[quad.Value]int) var validDeltas []graph.Delta if ignoreOpts.IgnoreDup || ignoreOpts.IgnoreMissing { validDeltas = make([]graph.Delta, 0, len(deltas)) } // Pre-check the existence condition. for _, d := range deltas { if d.Action != graph.Add && d.Action != graph.Delete { return &graph.DeltaError{Delta: d, Err: graph.ErrInvalidAction} } valid, err := qs.checkValidQuad(ctx, getKeyForQuad(d.Quad)) if err != nil { return &graph.DeltaError{Delta: d, Err: err} } switch d.Action { case graph.Add: if valid { if ignoreOpts.IgnoreDup { continue } else { return &graph.DeltaError{Delta: d, Err: graph.ErrQuadExists} } } case graph.Delete: if !valid { if ignoreOpts.IgnoreMissing { continue } else { return &graph.DeltaError{Delta: d, Err: graph.ErrQuadNotExist} } } } if validDeltas != nil { validDeltas = append(validDeltas, d) } var dn int if d.Action == graph.Add { dn = 1 } else { dn = -1 } ids[d.Quad.Subject] += dn ids[d.Quad.Object] += dn ids[d.Quad.Predicate] += dn if d.Quad.Label != nil { ids[d.Quad.Label] += dn } } if validDeltas != nil { deltas = validDeltas } if oids, err := qs.appendLog(ctx, deltas); err != nil { if i := len(oids); i < len(deltas) { return &graph.DeltaError{Delta: deltas[i], Err: err} } return &graph.DeltaError{Err: err} } // make sure to create all nodes before writing any quads // concurrent reads may observe broken quads in other case var gc []nosql.Key for name, dn := range ids { key := qs.nameToKey(name) err := qs.updateNodeBy(ctx, key, name, dn) if err != nil { return err } if dn < 0 { gc = append(gc, key) } } // gc nodes that has negative ref counter if err := qs.cleanupNodes(ctx, gc); err != nil { return err } for _, d := range deltas { err := qs.updateQuad(ctx, d.Quad, d.Action) if err != nil { return &graph.DeltaError{Delta: d, Err: err} } } return nil } func toDocumentValue(opt *Traits, v quad.Value) nosql.Document { if v == nil { return nil } var doc nosql.Document encPb := func() { qv := pquads.MakeValue(v) data, err := proto.Marshal(qv) if err != nil { panic(err) } doc[fldValPb] = nosql.Bytes(data) } switch d := v.(type) { case quad.String: doc = nosql.Document{fldValData: nosql.String(d)} case quad.IRI: doc = nosql.Document{fldValData: nosql.String(d), fldIRI: nosql.Bool(true)} case quad.BNode: doc = nosql.Document{fldValData: nosql.String(d), fldBNode: nosql.Bool(true)} case quad.TypedString: doc = nosql.Document{fldValData: nosql.String(d.Value), fldType: nosql.String(d.Type)} case quad.LangString: doc = nosql.Document{fldValData: nosql.String(d.Value), fldLang: nosql.String(d.Lang)} case quad.Int: doc = nosql.Document{fldValInt: nosql.Int(d)} if opt.Number32 { // store sortable string representation for range queries doc[fldValStrInt] = nosql.String(itos(int64(d))) encPb() } case quad.Float: doc = nosql.Document{fldValFloat: nosql.Float(d)} if opt.Number32 { encPb() } case quad.Bool: doc = nosql.Document{fldValBool: nosql.Bool(d)} case quad.Time: doc = nosql.Document{fldValTime: nosql.Time(time.Time(d).UTC())} default: encPb() } return nosql.Document{fldValue: doc} } func asInt(v nosql.Value) (nosql.Int, error) { var vi nosql.Int switch v := v.(type) { case nosql.Int: vi = v case nosql.Float: vi = nosql.Int(v) default: return 0, fmt.Errorf("unexpected type for int field: %T", v) } return vi, nil } func toQuadValue(opt *Traits, d nosql.Document) (quad.Value, error) { if len(d) == 0 { return nil, nil } var err error // prefer protobuf representation if v, ok := d[fldValPb]; ok { var b []byte switch v := v.(type) { case nosql.String: b, err = base64.StdEncoding.DecodeString(string(v)) case nosql.Bytes: b = []byte(v) default: err = fmt.Errorf("unexpected type for pb field: %T", v) } if err != nil { return nil, err } var p pquads.Value if err := proto.Unmarshal(b, &p); err != nil { return nil, fmt.Errorf("couldn't decode value: %v", err) } return p.ToNative(), nil } else if v, ok := d[fldValInt]; ok { if opt.Number32 { // parse from string, so we are confident that we will get exactly the same value if vs, ok := d[fldValStrInt].(nosql.String); ok { iv := quad.Int(stoi(string(vs))) return iv, nil } } vi, err := asInt(v) if err != nil { return nil, err } return quad.Int(vi), nil } else if v, ok := d[fldValFloat]; ok { var vf quad.Float switch v := v.(type) { case nosql.Int: vf = quad.Float(v) case nosql.Float: vf = quad.Float(v) default: return nil, fmt.Errorf("unexpected type for float field: %T", v) } return vf, nil } else if v, ok := d[fldValBool]; ok { var vb quad.Bool switch v := v.(type) { case nosql.Bool: vb = quad.Bool(v) default: return nil, fmt.Errorf("unexpected type for bool field: %T", v) } return vb, nil } else if v, ok := d[fldValTime]; ok { var vt quad.Time switch v := v.(type) { case nosql.Time: vt = quad.Time(v) case nosql.String: var t time.Time if err := t.UnmarshalJSON([]byte(`"` + string(v) + `"`)); err != nil { return nil, err } vt = quad.Time(t) default: return nil, fmt.Errorf("unexpected type for bool field: %T", v) } return vt, nil } vs, ok := d[fldValData].(nosql.String) if !ok { return nil, fmt.Errorf("unknown value format: %T", d[fldValData]) } if len(d) == 1 { return quad.String(vs), nil } if ok, _ := d[fldIRI].(nosql.Bool); ok { return quad.IRI(vs), nil } else if ok, _ := d[fldBNode].(nosql.Bool); ok { return quad.BNode(vs), nil } else if typ, ok := d[fldType].(nosql.String); ok { return quad.TypedString{Value: quad.String(vs), Type: quad.IRI(typ)}, nil } else if typ, ok := d[fldLang].(nosql.String); ok { return quad.LangString{Value: quad.String(vs), Lang: string(typ)}, nil } return nil, fmt.Errorf("unsupported value: %#v", d) } func (qs *QuadStore) Quad(val graph.Ref) (quad.Quad, error) { h := val.(QuadHash) var q quad.Quad var err error q.Subject, err = qs.NameOf(NodeHash(h.Get(quad.Subject))) if err != nil { return q, err } q.Predicate, err = qs.NameOf(NodeHash(h.Get(quad.Predicate))) if err != nil { return q, err } q.Object, err = qs.NameOf(NodeHash(h.Get(quad.Object))) if err != nil { return q, err } q.Label, err = qs.NameOf(NodeHash(h.Get(quad.Label))) return q, err } func (qs *QuadStore) QuadIterator(d quad.Direction, val graph.Ref) iterator.Shape { h, ok := val.(NodeHash) if !ok { return iterator.NewNull() } return qs.newLinksToIterator("quads", []Linkage{{Dir: d, Val: h}}) } func (qs *QuadStore) QuadIteratorSize(ctx context.Context, d quad.Direction, v graph.Ref) (refs.Size, error) { h, ok := v.(NodeHash) if !ok { return refs.Size{Value: 0, Exact: true}, nil } sz, err := qs.getSize("quads", linkageToFilters([]Linkage{{Dir: d, Val: h}})) if err != nil { return refs.Size{}, err } return refs.Size{ Value: sz, Exact: true, }, nil } func (qs *QuadStore) NodesAllIterator() iterator.Shape { return qs.newIterator("nodes") } func (qs *QuadStore) QuadsAllIterator() iterator.Shape { return qs.newIterator("quads") } func (qs *QuadStore) hashOf(s quad.Value) NodeHash { return NodeHash(hashOf(s)) } func (qs *QuadStore) ValueOf(s quad.Value) (graph.Ref, error) { if s == nil { return nil, nil } return qs.hashOf(s), nil } func (qs *QuadStore) NameOf(v graph.Ref) (quad.Value, error) { if v == nil { return nil, nil } else if v, ok := v.(refs.PreFetchedValue); ok { return v.NameOf(), nil } hash := v.(NodeHash) if hash == "" { return nil, nil } if val, ok := qs.ids.Get(string(hash)); ok { return val.(quad.Value), nil } nd, err := qs.db.FindByKey(context.TODO(), colNodes, hash.key()) if err == nosql.ErrNotFound { return nil, nil } else if err != nil { clog.Errorf("couldn't retrieve node %v: %v", v, err) return nil, err } dv, _ := nd[fldValue].(nosql.Document) qv, err := toQuadValue(&qs.opt, dv) if err != nil { clog.Errorf("couldn't convert node %v: %v", v, err) return nil, err } if id, _ := nd[fldHash].(nosql.String); id == nosql.String(hash) && qv != nil { qs.ids.Put(string(hash), qv) } return qv, nil } func (qs *QuadStore) Stats(ctx context.Context, exact bool) (graph.Stats, error) { // TODO(barakmich): Make size real; store it in the log, and retrieve it. nodes, err := qs.db.Query(colNodes).Count(ctx) if err != nil { return graph.Stats{}, err } quads, err := qs.db.Query(colQuads).Count(ctx) if err != nil { return graph.Stats{}, err } return graph.Stats{ Nodes: refs.Size{ Value: nodes, Exact: true, }, Quads: refs.Size{ Value: quads, Exact: true, }, }, nil } func (qs *QuadStore) Size() int64 { count, err := qs.db.Query(colQuads).Count(context.TODO()) if err != nil { clog.Errorf("%v", err) return 0 } return count } func (qs *QuadStore) Close() error { return qs.db.Close() } func (qs *QuadStore) QuadDirection(in graph.Ref, d quad.Direction) (graph.Ref, error) { return NodeHash(in.(QuadHash).Get(d)), nil } func (qs *QuadStore) getSize(col string, constraints []nosql.FieldFilter) (int64, error) { cacheKey := "" for _, c := range constraints { // FIXME cacheKey += fmt.Sprint(c.Path, c.Filter, c.Value) } key := col + cacheKey if val, ok := qs.sizes.Get(key); ok { return val.(int64), nil } q := qs.db.Query(col) if len(constraints) != 0 { q = q.WithFields(constraints...) } size, err := q.Count(context.TODO()) if err != nil { clog.Errorf("error getting size for iterator: %v", err) return -1, err } qs.sizes.Put(key, int64(size)) return int64(size), nil } ================================================ FILE: graph/nosql/shapes.go ================================================ package nosql import ( "context" "fmt" "math" "strconv" "github.com/hidal-go/hidalgo/legacy/nosql" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) var _ shape.Optimizer = (*QuadStore)(nil) func (qs *QuadStore) OptimizeShape(ctx context.Context, s shape.Shape) (shape.Shape, bool) { switch s := s.(type) { case shape.Quads: return qs.optimizeQuads(s) case shape.Filter: return qs.optimizeFilter(s) case shape.Page: return qs.optimizePage(s) case shape.Composite: if s2, opt := s.Simplify().Optimize(ctx, qs); opt { return s2, true } } return s, false } // Shape is a shape representing a documents query with filters type Shape struct { Collection string // name of the collection Filters []nosql.FieldFilter // filters to select documents Limit int64 // limits a number of documents } func (s Shape) BuildIterator(qs graph.QuadStore) iterator.Shape { db, ok := qs.(*QuadStore) if !ok { return iterator.NewError(fmt.Errorf("not a nosql database: %T", qs)) } return db.newIterator(s.Collection, s.Filters...) } func (s Shape) Optimize(ctx context.Context, r shape.Optimizer) (shape.Shape, bool) { return s, false } // Quads is a shape representing a quads query type Quads struct { Links []Linkage // filters to select quads Limit int64 // limits a number of documents } func (s Quads) BuildIterator(qs graph.QuadStore) iterator.Shape { db, ok := qs.(*QuadStore) if !ok { return iterator.NewError(fmt.Errorf("not a nosql database: %T", qs)) } return db.newLinksToIterator(colQuads, s.Links) } func (s Quads) Optimize(ctx context.Context, r shape.Optimizer) (shape.Shape, bool) { return s, false } const int64Adjust = 1 << 63 // itos serializes int64 into a sortable string 13 chars long. func itos(i int64) string { s := strconv.FormatUint(uint64(i)+int64Adjust, 32) const z = "0000000000000" return z[len(s):] + s } // stoi de-serializes int64 from a sortable string 13 chars long. func stoi(s string) int64 { ret, err := strconv.ParseUint(s, 32, 64) if err != nil { //TODO handle error? return 0 } return int64(ret - int64Adjust) } func toFieldFilter(opt *Traits, c shape.Comparison) ([]nosql.FieldFilter, bool) { var op nosql.FilterOp switch c.Op { case iterator.CompareGT: op = nosql.GT case iterator.CompareGTE: op = nosql.GTE case iterator.CompareLT: op = nosql.LT case iterator.CompareLTE: op = nosql.LTE default: return nil, false } fieldPath := func(s string) []string { return []string{fldValue, s} } var filters []nosql.FieldFilter switch v := c.Val.(type) { case quad.String: filters = []nosql.FieldFilter{ {Path: fieldPath(fldValData), Filter: op, Value: nosql.String(v)}, {Path: fieldPath(fldIRI), Filter: nosql.NotEqual, Value: nosql.Bool(true)}, {Path: fieldPath(fldBNode), Filter: nosql.NotEqual, Value: nosql.Bool(true)}, } case quad.IRI: filters = []nosql.FieldFilter{ {Path: fieldPath(fldValData), Filter: op, Value: nosql.String(v)}, {Path: fieldPath(fldIRI), Filter: nosql.Equal, Value: nosql.Bool(true)}, } case quad.BNode: filters = []nosql.FieldFilter{ {Path: fieldPath(fldValData), Filter: op, Value: nosql.String(v)}, {Path: fieldPath(fldBNode), Filter: nosql.Equal, Value: nosql.Bool(true)}, } case quad.Int: if opt.Number32 && (v < math.MinInt32 || v > math.MaxInt32) { // switch to range on string values filters = []nosql.FieldFilter{ {Path: fieldPath(fldValStrInt), Filter: op, Value: nosql.String(itos(int64(v)))}, } } else { filters = []nosql.FieldFilter{ {Path: fieldPath(fldValInt), Filter: op, Value: nosql.Int(v)}, } } case quad.Float: filters = []nosql.FieldFilter{ {Path: fieldPath(fldValFloat), Filter: op, Value: nosql.Float(v)}, } case quad.Time: filters = []nosql.FieldFilter{ {Path: fieldPath(fldValTime), Filter: op, Value: nosql.Time(v)}, } default: return nil, false } return filters, true } func (qs *QuadStore) optimizeFilter(s shape.Filter) (shape.Shape, bool) { if _, ok := s.From.(shape.AllNodes); !ok { return s, false } var ( filters []nosql.FieldFilter left []shape.ValueFilter ) fieldPath := func(s string) []string { return []string{fldValue, s} } for _, f := range s.Filters { switch f := f.(type) { case shape.Comparison: if fld, ok := toFieldFilter(&qs.opt, f); ok { filters = append(filters, fld...) continue } case shape.Wildcard: filters = append(filters, []nosql.FieldFilter{ {Path: fieldPath(fldValData), Filter: nosql.Regexp, Value: nosql.String(f.Regexp())}, }...) continue case shape.Regexp: filters = append(filters, []nosql.FieldFilter{ {Path: fieldPath(fldValData), Filter: nosql.Regexp, Value: nosql.String(f.Re.String())}, }...) if !f.Refs { filters = append(filters, []nosql.FieldFilter{ {Path: fieldPath(fldIRI), Filter: nosql.NotEqual, Value: nosql.Bool(true)}, {Path: fieldPath(fldBNode), Filter: nosql.NotEqual, Value: nosql.Bool(true)}, }...) } continue } left = append(left, f) } if len(filters) == 0 { return s, false } var ns shape.Shape = Shape{Collection: colNodes, Filters: filters} if len(left) != 0 { ns = shape.Filter{From: ns, Filters: left} } return ns, true } func (qs *QuadStore) optimizeQuads(s shape.Quads) (shape.Shape, bool) { var ( links []Linkage left []shape.QuadFilter ) for _, f := range s { if v, ok := shape.One(f.Values); ok { if h, ok := v.(NodeHash); ok { links = append(links, Linkage{Dir: f.Dir, Val: h}) continue } } left = append(left, f) } if len(links) == 0 { return s, false } var ns shape.Shape = Quads{Links: links} if len(left) != 0 { ns = shape.Intersect{ns, shape.Quads(left)} } return s, true } func (qs *QuadStore) optimizePage(s shape.Page) (shape.Shape, bool) { if s.Skip != 0 { return s, false } switch f := s.From.(type) { case shape.AllNodes: return Shape{Collection: colNodes, Limit: s.Limit}, false case Shape: s.ApplyPage(shape.Page{Limit: f.Limit}) f.Limit = s.Limit return f, true case Quads: s.ApplyPage(shape.Page{Limit: f.Limit}) f.Limit = s.Limit return f, true } return s, false } ================================================ FILE: graph/nosql/value_test.go ================================================ package nosql import ( "math" "sort" "testing" ) func TestIntStr(t *testing.T) { var testS []string testI := []int64{ 120000, -4, 88, 0, -7000000, 88, math.MaxInt64 - 1, math.MaxInt64, math.MinInt64, math.MinInt64 + 1, } for _, v := range testI { testS = append(testS, itos(v)) } sort.Strings(testS) sort.Slice(testI, func(i, j int) bool { return testI[i] < testI[j] }) for k, v := range testS { r := stoi(v) if r != testI[k] { t.Errorf("Sorting of stringed int64s wrong: %v %v %v", k, r, testI[k]) } } } ================================================ FILE: graph/proto/primitive.pb.go ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 // protoc v4.23.4 // source: primitive.proto package proto import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type PrimitiveType int32 const ( PrimitiveType_LINK PrimitiveType = 0 PrimitiveType_IRI PrimitiveType = 1 PrimitiveType_STRING PrimitiveType = 2 PrimitiveType_BNODE PrimitiveType = 3 PrimitiveType_TYPED_STR PrimitiveType = 4 PrimitiveType_LANG_STR PrimitiveType = 5 PrimitiveType_INT PrimitiveType = 6 PrimitiveType_FLOAT PrimitiveType = 7 PrimitiveType_BOOL PrimitiveType = 8 PrimitiveType_TIMESTAMP PrimitiveType = 9 ) // Enum value maps for PrimitiveType. var ( PrimitiveType_name = map[int32]string{ 0: "LINK", 1: "IRI", 2: "STRING", 3: "BNODE", 4: "TYPED_STR", 5: "LANG_STR", 6: "INT", 7: "FLOAT", 8: "BOOL", 9: "TIMESTAMP", } PrimitiveType_value = map[string]int32{ "LINK": 0, "IRI": 1, "STRING": 2, "BNODE": 3, "TYPED_STR": 4, "LANG_STR": 5, "INT": 6, "FLOAT": 7, "BOOL": 8, "TIMESTAMP": 9, } ) func (x PrimitiveType) Enum() *PrimitiveType { p := new(PrimitiveType) *p = x return p } func (x PrimitiveType) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } func (PrimitiveType) Descriptor() protoreflect.EnumDescriptor { return file_primitive_proto_enumTypes[0].Descriptor() } func (PrimitiveType) Type() protoreflect.EnumType { return &file_primitive_proto_enumTypes[0] } func (x PrimitiveType) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } // Deprecated: Use PrimitiveType.Descriptor instead. func (PrimitiveType) EnumDescriptor() ([]byte, []int) { return file_primitive_proto_rawDescGZIP(), []int{0} } type Primitive struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Subject uint64 `protobuf:"varint,2,opt,name=Subject,proto3" json:"Subject,omitempty"` Predicate uint64 `protobuf:"varint,3,opt,name=Predicate,proto3" json:"Predicate,omitempty"` Object uint64 `protobuf:"varint,4,opt,name=Object,proto3" json:"Object,omitempty"` Label uint64 `protobuf:"varint,5,opt,name=Label,proto3" json:"Label,omitempty"` Replaces uint64 `protobuf:"varint,6,opt,name=Replaces,proto3" json:"Replaces,omitempty"` Timestamp int64 `protobuf:"varint,7,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` Value []byte `protobuf:"bytes,8,opt,name=Value,proto3" json:"Value,omitempty"` Deleted bool `protobuf:"varint,9,opt,name=Deleted,proto3" json:"Deleted,omitempty"` } func (x *Primitive) Reset() { *x = Primitive{} if protoimpl.UnsafeEnabled { mi := &file_primitive_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *Primitive) String() string { return protoimpl.X.MessageStringOf(x) } func (*Primitive) ProtoMessage() {} func (x *Primitive) ProtoReflect() protoreflect.Message { mi := &file_primitive_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use Primitive.ProtoReflect.Descriptor instead. func (*Primitive) Descriptor() ([]byte, []int) { return file_primitive_proto_rawDescGZIP(), []int{0} } func (x *Primitive) GetID() uint64 { if x != nil { return x.ID } return 0 } func (x *Primitive) GetSubject() uint64 { if x != nil { return x.Subject } return 0 } func (x *Primitive) GetPredicate() uint64 { if x != nil { return x.Predicate } return 0 } func (x *Primitive) GetObject() uint64 { if x != nil { return x.Object } return 0 } func (x *Primitive) GetLabel() uint64 { if x != nil { return x.Label } return 0 } func (x *Primitive) GetReplaces() uint64 { if x != nil { return x.Replaces } return 0 } func (x *Primitive) GetTimestamp() int64 { if x != nil { return x.Timestamp } return 0 } func (x *Primitive) GetValue() []byte { if x != nil { return x.Value } return nil } func (x *Primitive) GetDeleted() bool { if x != nil { return x.Deleted } return false } var File_primitive_proto protoreflect.FileDescriptor var file_primitive_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x70, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xeb, 0x01, 0x0a, 0x09, 0x50, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x2a, 0x83, 0x01, 0x0a, 0x0d, 0x50, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x49, 0x4e, 0x4b, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x52, 0x49, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x4e, 0x4f, 0x44, 0x45, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x44, 0x5f, 0x53, 0x54, 0x52, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x4c, 0x41, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x52, 0x10, 0x05, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x06, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x07, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x08, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x09, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x61, 0x79, 0x6c, 0x65, 0x79, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x63, 0x61, 0x79, 0x6c, 0x65, 0x79, 0x2f, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_primitive_proto_rawDescOnce sync.Once file_primitive_proto_rawDescData = file_primitive_proto_rawDesc ) func file_primitive_proto_rawDescGZIP() []byte { file_primitive_proto_rawDescOnce.Do(func() { file_primitive_proto_rawDescData = protoimpl.X.CompressGZIP(file_primitive_proto_rawDescData) }) return file_primitive_proto_rawDescData } var file_primitive_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_primitive_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_primitive_proto_goTypes = []any{ (PrimitiveType)(0), // 0: proto.PrimitiveType (*Primitive)(nil), // 1: proto.Primitive } var file_primitive_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type 0, // [0:0] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name } func init() { file_primitive_proto_init() } func file_primitive_proto_init() { if File_primitive_proto != nil { return } if !protoimpl.UnsafeEnabled { file_primitive_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Primitive); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_primitive_proto_rawDesc, NumEnums: 1, NumMessages: 1, NumExtensions: 0, NumServices: 0, }, GoTypes: file_primitive_proto_goTypes, DependencyIndexes: file_primitive_proto_depIdxs, EnumInfos: file_primitive_proto_enumTypes, MessageInfos: file_primitive_proto_msgTypes, }.Build() File_primitive_proto = out.File file_primitive_proto_rawDesc = nil file_primitive_proto_goTypes = nil file_primitive_proto_depIdxs = nil } ================================================ FILE: graph/proto/primitive.proto ================================================ // Copyright 2016 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package proto; option go_package = "github.com/cayleygraph/cayley/graph/proto"; message Primitive { uint64 ID = 1; uint64 Subject = 2; uint64 Predicate = 3; uint64 Object = 4; uint64 Label = 5; uint64 Replaces = 6; int64 Timestamp = 7; bytes Value = 8; bool Deleted = 9; } enum PrimitiveType { LINK = 0; IRI = 1; STRING = 2; BNODE = 3; TYPED_STR = 4; LANG_STR = 5; INT = 6; FLOAT = 7; BOOL = 8; TIMESTAMP = 9; } ================================================ FILE: graph/proto/primitive_helpers.go ================================================ package proto import "github.com/cayleygraph/quad" //go:generate protoc --go_opt=paths=source_relative --proto_path=. --go_out=. primitive.proto func (p *Primitive) GetDirection(d quad.Direction) uint64 { switch d { case quad.Subject: return p.Subject case quad.Predicate: return p.Predicate case quad.Object: return p.Object case quad.Label: return p.Label } panic("unknown direction") } func (p *Primitive) SetDirection(d quad.Direction, v uint64) { switch d { case quad.Subject: p.Subject = v case quad.Predicate: p.Predicate = v case quad.Object: p.Object = v case quad.Label: p.Label = v } } func (p *Primitive) IsNode() bool { return len(p.Value) != 0 } func (p *Primitive) Key() interface{} { return p.ID } func (p *Primitive) IsSameLink(q *Primitive) bool { return p.Subject == q.Subject && p.Predicate == q.Predicate && p.Object == q.Object && p.Label == q.Label } ================================================ FILE: graph/proto/serializations.pb.go ================================================ // Copyright 2015 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 // protoc v4.23.4 // source: serializations.proto package proto import ( pquads "github.com/cayleygraph/quad/pquads" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type LogDelta struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Quad *pquads.Quad `protobuf:"bytes,2,opt,name=Quad,proto3" json:"Quad,omitempty"` Action int32 `protobuf:"varint,3,opt,name=Action,proto3" json:"Action,omitempty"` Timestamp int64 `protobuf:"varint,4,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` } func (x *LogDelta) Reset() { *x = LogDelta{} if protoimpl.UnsafeEnabled { mi := &file_serializations_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *LogDelta) String() string { return protoimpl.X.MessageStringOf(x) } func (*LogDelta) ProtoMessage() {} func (x *LogDelta) ProtoReflect() protoreflect.Message { mi := &file_serializations_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use LogDelta.ProtoReflect.Descriptor instead. func (*LogDelta) Descriptor() ([]byte, []int) { return file_serializations_proto_rawDescGZIP(), []int{0} } func (x *LogDelta) GetID() uint64 { if x != nil { return x.ID } return 0 } func (x *LogDelta) GetQuad() *pquads.Quad { if x != nil { return x.Quad } return nil } func (x *LogDelta) GetAction() int32 { if x != nil { return x.Action } return 0 } func (x *LogDelta) GetTimestamp() int64 { if x != nil { return x.Timestamp } return 0 } type HistoryEntry struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields History []uint64 `protobuf:"varint,1,rep,packed,name=History,proto3" json:"History,omitempty"` } func (x *HistoryEntry) Reset() { *x = HistoryEntry{} if protoimpl.UnsafeEnabled { mi := &file_serializations_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *HistoryEntry) String() string { return protoimpl.X.MessageStringOf(x) } func (*HistoryEntry) ProtoMessage() {} func (x *HistoryEntry) ProtoReflect() protoreflect.Message { mi := &file_serializations_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use HistoryEntry.ProtoReflect.Descriptor instead. func (*HistoryEntry) Descriptor() ([]byte, []int) { return file_serializations_proto_rawDescGZIP(), []int{1} } func (x *HistoryEntry) GetHistory() []uint64 { if x != nil { return x.History } return nil } type NodeData struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Size int64 `protobuf:"varint,2,opt,name=Size,proto3" json:"Size,omitempty"` Value *pquads.Value `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` } func (x *NodeData) Reset() { *x = NodeData{} if protoimpl.UnsafeEnabled { mi := &file_serializations_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } func (x *NodeData) String() string { return protoimpl.X.MessageStringOf(x) } func (*NodeData) ProtoMessage() {} func (x *NodeData) ProtoReflect() protoreflect.Message { mi := &file_serializations_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use NodeData.ProtoReflect.Descriptor instead. func (*NodeData) Descriptor() ([]byte, []int) { return file_serializations_proto_rawDescGZIP(), []int{2} } func (x *NodeData) GetName() string { if x != nil { return x.Name } return "" } func (x *NodeData) GetSize() int64 { if x != nil { return x.Size } return 0 } func (x *NodeData) GetValue() *pquads.Value { if x != nil { return x.Value } return nil } var File_serializations_proto protoreflect.FileDescriptor var file_serializations_proto_rawDesc = []byte{ 0x0a, 0x14, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0b, 0x71, 0x75, 0x61, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x72, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x04, 0x51, 0x75, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x71, 0x75, 0x61, 0x64, 0x73, 0x2e, 0x51, 0x75, 0x61, 0x64, 0x52, 0x04, 0x51, 0x75, 0x61, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x28, 0x0a, 0x0c, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x22, 0x57, 0x0a, 0x08, 0x4e, 0x6f, 0x64, 0x65, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x71, 0x75, 0x61, 0x64, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x61, 0x79, 0x6c, 0x65, 0x79, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x63, 0x61, 0x79, 0x6c, 0x65, 0x79, 0x2f, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( file_serializations_proto_rawDescOnce sync.Once file_serializations_proto_rawDescData = file_serializations_proto_rawDesc ) func file_serializations_proto_rawDescGZIP() []byte { file_serializations_proto_rawDescOnce.Do(func() { file_serializations_proto_rawDescData = protoimpl.X.CompressGZIP(file_serializations_proto_rawDescData) }) return file_serializations_proto_rawDescData } var file_serializations_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_serializations_proto_goTypes = []any{ (*LogDelta)(nil), // 0: proto.LogDelta (*HistoryEntry)(nil), // 1: proto.HistoryEntry (*NodeData)(nil), // 2: proto.NodeData (*pquads.Quad)(nil), // 3: pquads.Quad (*pquads.Value)(nil), // 4: pquads.Value } var file_serializations_proto_depIdxs = []int32{ 3, // 0: proto.LogDelta.Quad:type_name -> pquads.Quad 4, // 1: proto.NodeData.value:type_name -> pquads.Value 2, // [2:2] is the sub-list for method output_type 2, // [2:2] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name } func init() { file_serializations_proto_init() } func file_serializations_proto_init() { if File_serializations_proto != nil { return } if !protoimpl.UnsafeEnabled { file_serializations_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*LogDelta); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_serializations_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*HistoryEntry); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } file_serializations_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*NodeData); i { case 0: return &v.state case 1: return &v.sizeCache case 2: return &v.unknownFields default: return nil } } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_serializations_proto_rawDesc, NumEnums: 0, NumMessages: 3, NumExtensions: 0, NumServices: 0, }, GoTypes: file_serializations_proto_goTypes, DependencyIndexes: file_serializations_proto_depIdxs, MessageInfos: file_serializations_proto_msgTypes, }.Build() File_serializations_proto = out.File file_serializations_proto_rawDesc = nil file_serializations_proto_goTypes = nil file_serializations_proto_depIdxs = nil } ================================================ FILE: graph/proto/serializations.proto ================================================ // Copyright 2015 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package proto; option go_package = "github.com/cayleygraph/cayley/graph/proto"; import "quads.proto"; message LogDelta { uint64 ID = 1; pquads.Quad Quad = 2; int32 Action = 3; int64 Timestamp = 4; } message HistoryEntry { repeated uint64 History = 1; } message NodeData { string Name = 1; int64 Size = 2; pquads.Value value = 3; } ================================================ FILE: graph/proto/serializations_helpers.go ================================================ package proto import ( "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/pquads" ) //go:generate curl -LO https://github.com/cayleygraph/quad/raw/v1.3.0/pquads/quads.proto //go:generate protoc --go_opt=paths=source_relative --proto_path=. --go_out=. serializations.proto // GetNativeValue returns the value stored in Node. func (m *NodeData) GetNativeValue() quad.Value { if m == nil { return nil } else if m.Value == nil { if m.Name == "" { return nil } return quad.Raw(m.Name) } return m.Value.ToNative() } func (m *NodeData) Upgrade() { if m.Value == nil { m.Value = pquads.MakeValue(quad.Raw(m.Name)) m.Name = "" } } ================================================ FILE: graph/quadstore.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph // Defines the QuadStore interface. Every backing store must implement at // least this interface. // // Most of these are pretty straightforward. As long as we can surface this // interface, the rest of the stack will "just work" and we can connect to any // quad backing store we prefer. import ( "context" "errors" "fmt" "reflect" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/quad" ) // Ref defines an opaque "quad store reference" type. However the backend wishes // to implement it, a Ref is merely a token to a quad or a node that the // backing store itself understands, and the base iterators pass around. // // For example, in a very traditional, graphd-style graph, these are int64s // (guids of the primitives). In a very direct sort of graph, these could be // pointers to structs, or merely quads, or whatever works best for the // backing store. // // These must be comparable, or return a comparable version on Key. type Ref = refs.Ref func ValuesOf(ctx context.Context, qs refs.Namer, vals []Ref) ([]quad.Value, error) { return refs.ValuesOf(ctx, qs, vals) } func RefsOf(ctx context.Context, qs refs.Namer, nodes []quad.Value) ([]Ref, error) { return refs.RefsOf(ctx, qs, nodes) } type QuadIndexer interface { // Given an opaque token, returns the quad for that token from the store. Quad(Ref) (quad.Quad, error) // Given a direction and a token, creates an iterator of links which have // that node token in that directional field. QuadIterator(quad.Direction, Ref) iterator.Shape // QuadIteratorSize returns an estimated size of an iterator. QuadIteratorSize(ctx context.Context, d quad.Direction, v Ref) (refs.Size, error) // Convenience function for speed. Given a quad token and a direction // return the node token for that direction. Sometimes, a QuadStore // can do this without going all the way to the backing store, and // gives the QuadStore the opportunity to make this optimization. // // Iterators will call this. At worst, a valid implementation is // // qs.ValueOf(qs.Quad(id).Get(dir)) // QuadDirection(id Ref, d quad.Direction) (Ref, error) // Stats returns the number of nodes and quads currently stored. // Exact flag controls the correctness of the value. It can be an estimation, or a precise calculation. // The quadstore may have a fast way of retrieving the precise stats, in this case it may ignore 'exact' // flag and always return correct stats (with an appropriate flags set in the output). Stats(ctx context.Context, exact bool) (Stats, error) } // Stats of a graph. type Stats struct { Nodes refs.Size // number of nodes Quads refs.Size // number of quads } type QuadStore interface { refs.Namer QuadIndexer // The only way in is through building a transaction, which // is done by a replication strategy. ApplyDeltas(in []Delta, opts IgnoreOpts) error // NewQuadWriter starts a batch quad import process. // The order of changes is not guaranteed, neither is the order and result of concurrent ApplyDeltas. NewQuadWriter() (quad.WriteCloser, error) // Returns an iterator enumerating all nodes in the graph. NodesAllIterator() iterator.Shape // Returns an iterator enumerating all links in the graph. QuadsAllIterator() iterator.Shape // Close the quad store and clean up. (Flush to disk, cleanly // sever connections, etc) Close() error } type Options map[string]interface{} var ( typeInt = reflect.TypeOf(int(0)) ) func (d Options) IntKey(key string, def int) (int, error) { if val, ok := d[key]; ok { if reflect.TypeOf(val).ConvertibleTo(typeInt) { i := reflect.ValueOf(val).Convert(typeInt).Int() return int(i), nil } return def, fmt.Errorf("Invalid %s parameter type from config: %T", key, val) } return def, nil } func (d Options) StringKey(key string, def string) (string, error) { if val, ok := d[key]; ok { if v, ok := val.(string); ok { return v, nil } return def, fmt.Errorf("Invalid %s parameter type from config: %T", key, val) } return def, nil } func (d Options) BoolKey(key string, def bool) (bool, error) { if val, ok := d[key]; ok { if v, ok := val.(bool); ok { return v, nil } return def, fmt.Errorf("Invalid %s parameter type from config: %T", key, val) } return def, nil } var ( ErrDatabaseExists = errors.New("quadstore: cannot init; database already exists") ErrNotInitialized = errors.New("quadstore: not initialized") ) ================================================ FILE: graph/quadwriter.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph // Defines the interface for consistent replication of a graph instance. // // Separate from the backend, this dictates how individual quads get // identified and replicated consistently across (potentially) multiple // instances. The simplest case is to keep an append-only log of quad // changes. import ( "context" "errors" "io" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/graph/iterator" ) type Procedure int8 func (p Procedure) String() string { switch p { case +1: return "add" case -1: return "delete" default: return "invalid" } } // The different types of actions a transaction can do. const ( Add Procedure = +1 Delete Procedure = -1 ) type Delta struct { Quad quad.Quad Action Procedure } // Unwrap returns an original QuadStore value if it was wrapped by Handle. // This prevents shadowing of optional interface implementations. func Unwrap(qs QuadStore) QuadStore { if h, ok := qs.(*Handle); ok { return h.QuadStore } return qs } type Handle struct { QuadStore QuadWriter } type IgnoreOpts struct { IgnoreDup, IgnoreMissing bool } func (h *Handle) Close() error { err := h.QuadWriter.Close() h.QuadStore.Close() return err } var ( ErrQuadExists = errors.New("quad exists") ErrQuadNotExist = errors.New("quad does not exist") ErrInvalidAction = errors.New("invalid action") ErrNodeNotExists = errors.New("node does not exist") ) // DeltaError records an error and the delta that caused it. type DeltaError struct { Delta Delta Err error } func (e *DeltaError) Error() string { if !e.Delta.Quad.IsValid() { return e.Err.Error() } return e.Delta.Action.String() + " " + e.Delta.Quad.String() + ": " + e.Err.Error() } func (e *DeltaError) Unwrap() error { return e.Err } // IsQuadExist returns whether an error is a DeltaError // with the Err field equal to ErrQuadExists. func IsQuadExist(err error) bool { return errors.Is(err, ErrQuadExists) } // IsQuadNotExist returns whether an error is a DeltaError // with the Err field equal to ErrQuadNotExist. func IsQuadNotExist(err error) bool { return errors.Is(err, ErrQuadNotExist) } // IsInvalidAction returns whether an error is a DeltaError // with the Err field equal to ErrInvalidAction. func IsInvalidAction(err error) bool { return errors.Is(err, ErrInvalidAction) } var ( // IgnoreDuplicates specifies whether duplicate quads // cause an error during loading or are ignored. IgnoreDuplicates = true // IgnoreMissing specifies whether missing quads // cause an error during deletion or are ignored. IgnoreMissing = false ) type QuadWriter interface { // AddQuad adds a quad to the store. AddQuad(quad.Quad) error // TODO(barakmich): Deprecate in favor of transaction. // AddQuadSet adds a set of quads to the store, atomically if possible. AddQuadSet([]quad.Quad) error // RemoveQuad removes a quad matching the given one from the database, // if it exists. Does nothing otherwise. RemoveQuad(quad.Quad) error // ApplyTransaction applies a set of quad changes. ApplyTransaction(*Transaction) error // RemoveNode removes all quads which have the given node as subject, predicate, object, or label. // // It returns ErrNodeNotExists if node is missing. RemoveNode(quad.Value) error // Close cleans up replication and closes the writing aspect of the database. Close() error } type NewQuadWriterFunc func(QuadStore, Options) (QuadWriter, error) var writerRegistry = make(map[string]NewQuadWriterFunc) func RegisterWriter(name string, newFunc NewQuadWriterFunc) { if _, found := writerRegistry[name]; found { panic("already registered QuadWriter " + name) } writerRegistry[name] = newFunc } func NewQuadWriter(name string, qs QuadStore, opts Options) (QuadWriter, error) { newFunc, hasNew := writerRegistry[name] if !hasNew { return nil, errors.New("replication: name '" + name + "' is not registered") } return newFunc(qs, opts) } func WriterMethods() []string { t := make([]string, 0, len(writerRegistry)) for n := range writerRegistry { t = append(t, n) } return t } type BatchWriter interface { quad.WriteCloser Flush() error } // NewWriter creates a quad writer for a given QuadStore. // // Caller must call Flush or Close to flush an internal buffer. func NewWriter(qs QuadWriter) BatchWriter { return &batchWriter{qs: qs} } type batchWriter struct { qs QuadWriter buf []quad.Quad } func (w *batchWriter) flushBuffer(force bool) error { if !force && len(w.buf) < quad.DefaultBatch { return nil } _, err := w.WriteQuads(w.buf) w.buf = w.buf[:0] return err } func (w *batchWriter) WriteQuad(q quad.Quad) error { if err := w.flushBuffer(false); err != nil { return err } w.buf = append(w.buf, q) return nil } func (w *batchWriter) WriteQuads(quads []quad.Quad) (int, error) { if err := w.qs.AddQuadSet(quads); err != nil { return 0, err } return len(quads), nil } func (w *batchWriter) Flush() error { return w.flushBuffer(true) } func (w *batchWriter) Close() error { return w.Flush() } // NewTxWriter creates a writer that applies a given procedures for all quads in stream. // If procedure is zero, Add operation will be used. func NewTxWriter(tx *Transaction, p Procedure) quad.Writer { if p == 0 { p = Add } return &txWriter{tx: tx, p: p} } type txWriter struct { tx *Transaction p Procedure } func (w *txWriter) WriteQuad(q quad.Quad) error { switch w.p { case Add: w.tx.AddQuad(q) case Delete: w.tx.RemoveQuad(q) default: return ErrInvalidAction } return nil } func (w *txWriter) WriteQuads(buf []quad.Quad) (int, error) { for i, q := range buf { if err := w.WriteQuad(q); err != nil { return i, err } } return len(buf), nil } // NewRemover creates a quad writer for a given QuadStore which removes quads instead of adding them. func NewRemover(qs QuadWriter) BatchWriter { return &removeWriter{qs: qs} } type removeWriter struct { qs QuadWriter } func (w *removeWriter) WriteQuad(q quad.Quad) error { return w.qs.RemoveQuad(q) } func (w *removeWriter) WriteQuads(quads []quad.Quad) (int, error) { tx := NewTransaction() for _, q := range quads { tx.RemoveQuad(q) } if err := w.qs.ApplyTransaction(tx); err != nil { return 0, err } return len(quads), nil } func (w *removeWriter) Flush() error { return nil // TODO: batch deletes automatically } func (w *removeWriter) Close() error { return nil } // NewQuadStoreReader creates a quad reader for a given QuadStore. func NewQuadStoreReader(qs QuadStore) quad.ReadSkipCloser { return NewResultReader(qs, nil) } // NewResultReader creates a quad reader for a given QuadStore and iterator. // If iterator is nil QuadsAllIterator will be used. // // Only quads returned by iterator's Result will be used. // // Iterator will be closed with the reader. func NewResultReader(qs QuadStore, it iterator.Scanner) quad.ReadSkipCloser { if it == nil { it = qs.QuadsAllIterator().Iterate() } return &quadReader{qs: qs, it: it} } type quadReader struct { qs QuadStore it iterator.Scanner } func (r *quadReader) ReadQuad() (quad.Quad, error) { if r.it.Next(context.TODO()) { return r.qs.Quad(r.it.Result()) } err := r.it.Err() if err == nil { err = io.EOF } return quad.Quad{}, err } func (r *quadReader) SkipQuad() error { if r.it.Next(context.TODO()) { return nil } if err := r.it.Err(); err != nil { return err } return io.EOF } func (r *quadReader) Close() error { return r.it.Close() } ================================================ FILE: graph/quadwriter_test.go ================================================ package graph import ( "errors" "testing" ) func TestIsQuadExist(t *testing.T) { tests := []struct { Err error Matches bool }{ {Err: nil, Matches: false}, {Err: errors.New("foo"), Matches: false}, {Err: ErrQuadExists, Matches: true}, {Err: &DeltaError{Err: errors.New("foo")}, Matches: false}, {Err: &DeltaError{Err: ErrQuadExists}, Matches: true}, } for i, test := range tests { if match := IsQuadExist(test.Err); test.Matches != match { t.Errorf("%d> unexpected match: %t", i, match) } } } func TestIsQuadNotExist(t *testing.T) { tests := []struct { Err error Matches bool }{ {Err: nil, Matches: false}, {Err: errors.New("foo"), Matches: false}, {Err: ErrQuadNotExist, Matches: true}, {Err: &DeltaError{Err: errors.New("foo")}, Matches: false}, {Err: &DeltaError{Err: ErrQuadNotExist}, Matches: true}, } for i, test := range tests { if match := IsQuadNotExist(test.Err); test.Matches != match { t.Errorf("%d> unexpected match: %t", i, match) } } } func TestIsInvalidAction(t *testing.T) { tests := []struct { Err error Matches bool }{ {Err: nil, Matches: false}, {Err: errors.New("foo"), Matches: false}, {Err: ErrInvalidAction, Matches: true}, {Err: &DeltaError{Err: errors.New("foo")}, Matches: false}, {Err: &DeltaError{Err: ErrInvalidAction}, Matches: true}, } for i, test := range tests { if match := IsInvalidAction(test.Err); test.Matches != match { t.Errorf("%d> unexpected match: %t", i, match) } } } ================================================ FILE: graph/refs/refs.go ================================================ package refs import ( "context" "encoding/hex" "fmt" "github.com/cayleygraph/quad" ) // Size of a graph (either in nodes or quads). type Size struct { Value int64 Exact bool } // Ref defines an opaque "quad store reference" type. However the backend wishes // to implement it, a Ref is merely a token to a quad or a node that the // backing store itself understands, and the base iterators pass around. // // For example, in a very traditional, graphd-style graph, these are int64s // (guids of the primitives). In a very direct sort of graph, these could be // pointers to structs, or merely quads, or whatever works best for the // backing store. // // These must be comparable, or return a comparable version on Key. type Ref interface { // Key returns a dynamic type that is comparable according to the Go language specification. // The returned value must be unique for each receiver value. Key() interface{} } type Namer interface { // Given a node ID, return the opaque token used by the QuadStore // to represent that id. Returns nil, nil on not found. ValueOf(quad.Value) (Ref, error) // Given an opaque token, return the node that it represents. // Returns nil, nil on not found. NameOf(Ref) (quad.Value, error) } type BatchNamer interface { ValuesOf(ctx context.Context, vals []Ref) ([]quad.Value, error) RefsOf(ctx context.Context, nodes []quad.Value) ([]Ref, error) } func HashOf(s quad.Value) (out ValueHash) { if s == nil { return } quad.HashTo(s, out[:]) return } var _ Ref = ValueHash{} // ValueHash is a hash of a single value. type ValueHash [quad.HashSize]byte func (h ValueHash) Valid() bool { return h != ValueHash{} } func (h ValueHash) Key() interface{} { return h } func (h ValueHash) String() string { if !h.Valid() { return "" } return hex.EncodeToString(h[:]) } // PreFetchedValue is an optional interface for graph.Ref to indicate that // quadstore has already loaded a value into memory. type PreFetchedValue interface { Ref NameOf() quad.Value } func PreFetched(v quad.Value) PreFetchedValue { return fetchedValue{v} } type fetchedValue struct { Val quad.Value } func (v fetchedValue) IsNode() bool { return true } func (v fetchedValue) NameOf() quad.Value { return v.Val } func (v fetchedValue) Key() interface{} { return v.Val } // ToKey prepares Ref to be stored inside maps, calling Key() if necessary. func ToKey(v Ref) interface{} { if v == nil { return nil } return v.Key() } var _ Ref = QuadHash{} type QuadHash struct { Subject ValueHash Predicate ValueHash Object ValueHash Label ValueHash } func (q QuadHash) Dirs() [4]ValueHash { return [4]ValueHash{ q.Subject, q.Predicate, q.Object, q.Label, } } func (q QuadHash) Key() interface{} { return q } func (q QuadHash) Get(d quad.Direction) ValueHash { switch d { case quad.Subject: return q.Subject case quad.Predicate: return q.Predicate case quad.Object: return q.Object case quad.Label: return q.Label } panic(fmt.Errorf("unknown direction: %v", d)) } func (q *QuadHash) Set(d quad.Direction, h ValueHash) { switch d { case quad.Subject: q.Subject = h case quad.Predicate: q.Predicate = h case quad.Object: q.Object = h case quad.Label: q.Label = h default: panic(fmt.Errorf("unknown direction: %v", d)) } } func ValuesOf(ctx context.Context, qs Namer, vals []Ref) ([]quad.Value, error) { if bq, ok := qs.(BatchNamer); ok { return bq.ValuesOf(ctx, vals) } out := make([]quad.Value, len(vals)) var err error for i, v := range vals { out[i], err = qs.NameOf(v) if err != nil { return nil, err } } return out, nil } func RefsOf(ctx context.Context, qs Namer, nodes []quad.Value) ([]Ref, error) { if bq, ok := qs.(BatchNamer); ok { return bq.RefsOf(ctx, nodes) } values := make([]Ref, len(nodes)) for i, node := range nodes { value, err := qs.ValueOf(node) if err != nil { return nil, err } if value == nil { return nil, fmt.Errorf("not found: %v", node) } values[i] = value } return values, nil } ================================================ FILE: graph/registry.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph import ( "fmt" "sort" ) var ( ErrQuadStoreNotRegistred = fmt.Errorf("this QuadStore is not registered") ErrQuadStoreNotPersistent = fmt.Errorf("cannot specify address for non-persistent backend") ErrOperationNotSupported = fmt.Errorf("this Operation is not supported") ) var storeRegistry = make(map[string]QuadStoreRegistration) type NewStoreFunc func(string, Options) (QuadStore, error) type InitStoreFunc func(string, Options) error type UpgradeStoreFunc func(string, Options) error type QuadStoreRegistration struct { NewFunc NewStoreFunc UpgradeFunc UpgradeStoreFunc InitFunc InitStoreFunc IsPersistent bool } func RegisterQuadStore(name string, register QuadStoreRegistration) { if register.NewFunc == nil { panic("NewFunc must not be nil") } // Register QuadStore with friendly name if _, found := storeRegistry[name]; found { panic(fmt.Sprintf("Already registered QuadStore %q.", name)) } storeRegistry[name] = register } func NewQuadStore(name string, dbpath string, opts Options) (QuadStore, error) { r, registered := storeRegistry[name] if !registered { return nil, ErrQuadStoreNotRegistred } else if dbpath != "" && !r.IsPersistent { return nil, ErrQuadStoreNotPersistent } return r.NewFunc(dbpath, opts) } func InitQuadStore(name string, dbpath string, opts Options) error { r, registered := storeRegistry[name] if !registered { return ErrQuadStoreNotRegistred } else if r.InitFunc == nil { return ErrOperationNotSupported } return r.InitFunc(dbpath, opts) } func UpgradeQuadStore(name string, dbpath string, opts Options) error { r, registered := storeRegistry[name] if !registered { return ErrQuadStoreNotRegistred } else if r.UpgradeFunc == nil { // return ErrOperationNotSupported return nil } return r.UpgradeFunc(dbpath, opts) } func IsRegistered(name string) bool { _, ok := storeRegistry[name] return ok } func IsPersistent(name string) bool { return storeRegistry[name].IsPersistent } func QuadStores() []string { t := make([]string, 0, len(storeRegistry)) for n := range storeRegistry { t = append(t, n) } sort.Strings(t) return t } ================================================ FILE: graph/sql/cockroach/cockroach.go ================================================ package cockroach import ( "bytes" "database/sql" "errors" "fmt" "strings" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" _ "github.com/jackc/pgx/v5/stdlib" // registers "pgx" driver "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" graphlog "github.com/cayleygraph/cayley/graph/log" csql "github.com/cayleygraph/cayley/graph/sql" ) const Type = "cockroach" func init() { csql.Register(Type, csql.Registration{ Driver: "pgx", HashType: `BYTEA`, BytesType: `BYTEA`, HorizonType: `BIGSERIAL`, TimeType: `timestamp with time zone`, NodesTableExtra: ` FAMILY fhash (hash), FAMILY frefs (refs), FAMILY fvalue (value, value_string, datatype, language, iri, bnode, value_int, value_bool, value_float, value_time) `, QueryDialect: csql.QueryDialect{ RegexpOp: "~", FieldQuote: func(name string) string { return pgx.Identifier{name}.Sanitize() }, Placeholder: func(n int) string { return fmt.Sprintf("$%d", n) }, }, NoForeignKeys: true, Error: convError, //Estimated: func(table string) string{ // return "SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='"+table+"';" //}, RunTx: runTxCockroach, TxRetry: retryTxCockroach, NoSchemaChangesInTx: true, }) } // AmbiguousCommitError represents an error that left a transaction in an // ambiguous state: unclear if it committed or not. type AmbiguousCommitError struct { Err error } func (e *AmbiguousCommitError) Error() string { return e.Err.Error() } func (e *AmbiguousCommitError) Unwrap() error { return e.Err } // retryTxCockroach runs the transaction and will retry in case of a retryable error. // https://www.cockroachlabs.com/docs/transactions.html#client-side-transaction-retries func retryTxCockroach(tx *sql.Tx, stmts func() error) error { // Specify that we intend to retry this txn in case of CockroachDB retryable // errors. if _, err := tx.Exec("SAVEPOINT cockroach_restart"); err != nil { return err } for { released := false err := stmts() if err == nil { // RELEASE acts like COMMIT in CockroachDB. We use it since it gives us an // opportunity to react to retryable errors, whereas tx.Commit() doesn't. released = true if _, err = tx.Exec("RELEASE SAVEPOINT cockroach_restart"); err == nil { return nil } } // We got an error; let's see if it's a retryable one and, if so, restart. We look // for either the standard PG errcode SerializationFailureError:40001 or the Cockroach extension // errcode RetriableError:CR000. The Cockroach extension has been removed server-side, but support // for it has been left here for now to maintain backwards compatibility. var pgErr *pgconn.PgError if !errors.As(err, &pgErr) { return err } if retryable := pgErr.Code == "CR000" || pgErr.Code == "40001"; !retryable { if released { err = &AmbiguousCommitError{err} } return err } if _, err = tx.Exec("ROLLBACK TO SAVEPOINT cockroach_restart"); err != nil { return err } } } func convError(err error) error { var e *pgconn.PgError if !errors.As(err, &e) { return err } switch e.Code { case "42P07": return graph.ErrDatabaseExists } return err } func convInsertError(err error) error { if err == nil { return nil } var pe *pgconn.PgError if errors.As(err, &pe) { if pe.Code == "23505" { // TODO: reference to delta return &graph.DeltaError{Err: graph.ErrQuadExists} } } return err } // runTxCockroach performs the node and quad updates in the provided transaction. // This is based on ../postgres/postgres.go:RunTx, but focuses on doing fewer insert statements, // since those are comparatively expensive for CockroachDB. func runTxCockroach(tx *sql.Tx, nodes []graphlog.NodeUpdate, quads []graphlog.QuadUpdate, opts graph.IgnoreOpts) error { // First, compile the sets of nodes, split by csql.ValueType. // Each of those will require a separate INSERT statement. type nodeEntry struct { refInc int values []interface{} // usually two, but sometimes three elements (includes hash) } nodeEntries := make(map[csql.ValueType][]nodeEntry) for _, n := range nodes { if n.RefInc < 0 { panic("unexpected node update") } nodeType, values, err := csql.NodeValues(csql.NodeHash{ValueHash: n.Hash}, n.Val) if err != nil { return err } nodeEntries[nodeType] = append(nodeEntries[nodeType], nodeEntry{ refInc: n.RefInc, values: values, }) } // Next, build and execute the INSERT statements for each type. for nodeType, entries := range nodeEntries { var query bytes.Buffer var allValues []interface{} valCols := nodeType.Columns() fmt.Fprintf(&query, "INSERT INTO nodes (refs, hash, %s) VALUES ", strings.Join(valCols, ", ")) ph := 1 // next placeholder counter for i, entry := range entries { if i > 0 { fmt.Fprint(&query, ", ") } fmt.Fprint(&query, "(") // sanity check if len(entry.values) != 1+len(valCols) { // +1 for hash, which is in values panic(fmt.Sprintf("internal error: %d entry values vs. %d value columns", len(entry.values), len(valCols))) } for j := 0; j < 1+len(entry.values); j++ { // +1 for refs if j > 0 { fmt.Fprint(&query, ", ") } fmt.Fprintf(&query, "$%d", ph) ph++ } fmt.Fprint(&query, ")") allValues = append(allValues, entry.refInc) allValues = append(allValues, entry.values...) } fmt.Fprint(&query, " ON CONFLICT (hash) DO UPDATE SET refs = nodes.refs + EXCLUDED.refs RETURNING NOTHING;") _, err := tx.Exec(query.String(), allValues...) err = convInsertError(err) if err != nil { clog.Errorf("couldn't exec node INSERT statement [%s]: %v", query.String(), err) return err } } // Now do the same thing with quads. // It is simpler because there's only one composite type to insert, // so only one INSERT statement is required. if len(quads) == 0 { return nil } var query bytes.Buffer var allValues []interface{} fmt.Fprintf(&query, "INSERT INTO quads (subject_hash, predicate_hash, object_hash, label_hash, ts) VALUES ") for i, d := range quads { if d.Del { panic("unexpected quad delete") } if i > 0 { fmt.Fprint(&query, ", ") } fmt.Fprintf(&query, "($%d, $%d, $%d, $%d, now())", 4*i+1, 4*i+2, 4*i+3, 4*i+4) allValues = append(allValues, csql.NodeHash{ValueHash: d.Quad.Subject}.SQLValue(), csql.NodeHash{ValueHash: d.Quad.Predicate}.SQLValue(), csql.NodeHash{ValueHash: d.Quad.Object}.SQLValue(), csql.NodeHash{ValueHash: d.Quad.Label}.SQLValue()) } if opts.IgnoreDup { fmt.Fprint(&query, " ON CONFLICT (subject_hash, predicate_hash, object_hash) DO NOTHING") // Only use RETURNING NOTHING when we're ignoring duplicates; // otherwise the error returned on duplicates will be different. fmt.Fprint(&query, " RETURNING NOTHING") } fmt.Fprint(&query, ";") _, err := tx.Exec(query.String(), allValues...) err = convInsertError(err) if err != nil { if _, ok := err.(*graph.DeltaError); !ok { clog.Errorf("couldn't exec quad INSERT statement [%s]: %v", query.String(), err) } return err } return nil } ================================================ FILE: graph/sql/cockroach/cockroach_test.go ================================================ //go:build docker // +build docker package cockroach import ( "context" "database/sql" "net" "strconv" "testing" "time" "github.com/jackc/pgx/v5" _ "github.com/jackc/pgx/v5/stdlib" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/sql/sqltest" "github.com/cayleygraph/cayley/internal/dock" ) func makeCockroach(t testing.TB) (string, graph.Options) { var conf dock.Config conf.Image = "cockroachdb/cockroach:latest-v24.1" conf.Cmd = []string{"start-single-node", "--insecure"} addr := dock.RunAndWait(t, conf, "26257", func(addr string) bool { host, portStr, err := net.SplitHostPort(addr) if err != nil { return false } port, err := strconv.Atoi(portStr) if err != nil { return false } cconf, err := pgx.ParseConfig("") if err != nil { return false } cconf.Host = host cconf.Port = uint16(port) cconf.User = "root" ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() conn, err := pgx.ConnectConfig(ctx, cconf) if err != nil { return false } conn.Close(ctx) return true }) addr = `postgresql://root@` + addr db, err := sql.Open("pgx", addr+`?sslmode=disable`) if err != nil { t.Fatal(err) } defer db.Close() const dbName = "cayley" if _, err = db.Exec("CREATE DATABASE " + dbName); err != nil { t.Fatal(err) } addr = addr + `/` + dbName + `?sslmode=disable` return addr, nil } var conf = &sqltest.Config{ TimeRound: true, TimeInMcs: true, } func TestCockroach(t *testing.T) { sqltest.TestAll(t, Type, makeCockroach, conf) } func BenchmarkCockroach(t *testing.B) { sqltest.BenchmarkAll(t, Type, makeCockroach, conf) } ================================================ FILE: graph/sql/database.go ================================================ package sql import ( "database/sql" "fmt" "strings" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/graph" graphlog "github.com/cayleygraph/cayley/graph/log" ) var types = make(map[string]Registration) func Register(name string, f Registration) { if f.Driver == "" { panic("no sql driver in type definition") } types[name] = f registerQuadStore(name, name) } type Registration struct { Driver string // sql driver to use on dial HashType string // type for hash fields BytesType string // type for binary fields TimeType string // type for datetime fields HorizonType string // type for horizon counter NodesTableExtra string // extra SQL to append to nodes table definition ConditionalIndexes bool // database supports conditional indexes FillFactor bool // database supports fill percent on indexes NoForeignKeys bool // database has no support for FKs CustomNullTime bool // driver doesn't support sql.NullTime QueryDialect NoOffsetWithoutLimit bool // SELECT ... OFFSET can be used only with LIMIT Error func(error) error // error conversion function Estimated func(table string) string // query that string that returns an estimated number of rows in table RunTx func(tx *sql.Tx, nodes []graphlog.NodeUpdate, quads []graphlog.QuadUpdate, opts graph.IgnoreOpts) error TxRetry func(tx *sql.Tx, stmts func() error) error NoSchemaChangesInTx bool } func (r Registration) nodesTable() string { htyp := r.HashType if htyp == "" { htyp = "BYTEA" } btyp := r.BytesType if btyp == "" { btyp = "BYTEA" } ttyp := r.TimeType if ttyp == "" { ttyp = "timestamp with time zone" } end := "\n);" if r.NodesTableExtra != "" { end = ",\n" + r.NodesTableExtra + end } return `CREATE TABLE nodes ( hash ` + htyp + ` PRIMARY KEY, refs INT NOT NULL, value ` + btyp + `, value_string TEXT, datatype TEXT, language TEXT, iri BOOLEAN, bnode BOOLEAN, value_int BIGINT, value_bool BOOLEAN, value_float double precision, value_time ` + ttyp + end } func (r Registration) quadsTable() string { htyp := r.HashType if htyp == "" { htyp = "BYTEA" } hztyp := r.HorizonType if hztyp == "" { hztyp = "SERIAL" } return `CREATE TABLE quads ( horizon ` + hztyp + ` PRIMARY KEY, subject_hash ` + htyp + ` NOT NULL, predicate_hash ` + htyp + ` NOT NULL, object_hash ` + htyp + ` NOT NULL, label_hash ` + htyp + `, ts timestamp );` } func (r Registration) quadIndexes(options graph.Options) []string { indexes := make([]string, 0, 10) if r.ConditionalIndexes { indexes = append(indexes, `CREATE UNIQUE INDEX spo_unique ON quads (subject_hash, predicate_hash, object_hash) WHERE label_hash IS NULL;`, `CREATE UNIQUE INDEX spol_unique ON quads (subject_hash, predicate_hash, object_hash, label_hash) WHERE label_hash IS NOT NULL;`, ) } else { indexes = append(indexes, `CREATE UNIQUE INDEX spo_unique ON quads (subject_hash, predicate_hash, object_hash);`, `CREATE UNIQUE INDEX spol_unique ON quads (subject_hash, predicate_hash, object_hash, label_hash);`, ) } if !r.NoForeignKeys { indexes = append(indexes, `ALTER TABLE quads ADD CONSTRAINT subject_hash_fk FOREIGN KEY (subject_hash) REFERENCES nodes (hash);`, `ALTER TABLE quads ADD CONSTRAINT predicate_hash_fk FOREIGN KEY (predicate_hash) REFERENCES nodes (hash);`, `ALTER TABLE quads ADD CONSTRAINT object_hash_fk FOREIGN KEY (object_hash) REFERENCES nodes (hash);`, `ALTER TABLE quads ADD CONSTRAINT label_hash_fk FOREIGN KEY (label_hash) REFERENCES nodes (hash);`, ) } quadIndexes := [][3]quad.Direction{ {quad.Subject, quad.Predicate, quad.Object}, {quad.Object, quad.Predicate, quad.Subject}, {quad.Predicate, quad.Object, quad.Subject}, {quad.Object, quad.Subject, quad.Predicate}, } factor, _ := options.IntKey("db_fill_factor", 50) for _, ind := range quadIndexes { var ( name string cols []string ) for _, d := range ind { name += string(d.Prefix()) cols = append(cols, d.String()+"_hash") } q := fmt.Sprintf(`CREATE INDEX %s_index ON quads (%s)`, name, strings.Join(cols, ", ")) if r.FillFactor { q += fmt.Sprintf(" WITH (FILLFACTOR = %d)", factor) } indexes = append(indexes, q+";") } return indexes } ================================================ FILE: graph/sql/iterator.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package sql import ( "context" "database/sql" "fmt" "strings" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) var _ shape.Optimizer = (*QuadStore)(nil) func (qs *QuadStore) OptimizeShape(ctx context.Context, s shape.Shape) (shape.Shape, bool) { return qs.opt.OptimizeShape(ctx, s) } func (qs *QuadStore) prepareQuery(s Shape) (string, []interface{}) { args := s.Args() vals := make([]interface{}, 0, len(args)) for _, a := range args { vals = append(vals, a.SQLValue()) } b := NewBuilder(qs.flavor.QueryDialect) qu := s.SQL(b) return qu, vals } func (qs *QuadStore) QueryRow(ctx context.Context, s Shape) *sql.Row { qu, vals := qs.prepareQuery(s) return qs.db.QueryRowContext(ctx, qu, vals...) } func (qs *QuadStore) Query(ctx context.Context, s Shape) (*sql.Rows, error) { qu, vals := qs.prepareQuery(s) rows, err := qs.db.QueryContext(ctx, qu, vals...) if err != nil { return nil, fmt.Errorf("sql query failed: %v\nquery: %v", err, qu) } return rows, nil } func (qs *QuadStore) newIterator(s Select) *Iterator { return &Iterator{ qs: qs, query: s, } } type Iterator struct { qs *QuadStore query Select err error } func (it *Iterator) Iterate() iterator.Scanner { return it.qs.newIteratorNext(it.query) } func (it *Iterator) Lookup() iterator.Index { return it.qs.newIteratorContains(it.query) } func (it *Iterator) Stats(ctx context.Context) (iterator.Costs, error) { sz, err := it.getSize(ctx) return iterator.Costs{ NextCost: 1, ContainsCost: 10, Size: sz, }, err } func (it *Iterator) estimateSize(ctx context.Context) int64 { if it.query.Limit > 0 { return it.query.Limit } st, err := it.qs.Stats(ctx, false) if err != nil && it.err == nil { it.err = err } return st.Quads.Value } func (it *Iterator) getSize(ctx context.Context) (refs.Size, error) { sz, err := it.qs.querySize(ctx, it.query) if err != nil { it.err = err return refs.Size{Value: it.estimateSize(ctx), Exact: false}, err } return sz, nil } func (it *Iterator) Optimize(ctx context.Context) (iterator.Shape, bool) { return it, false } func (it *Iterator) SubIterators() []iterator.Shape { return nil } func (it *Iterator) String() string { return it.query.SQL(NewBuilder(it.qs.flavor.QueryDialect)) } func newIteratorBase(qs *QuadStore, s Select) iteratorBase { return iteratorBase{ qs: qs, query: s, } } type iteratorBase struct { qs *QuadStore query Select cols []string cind map[quad.Direction]int err error res graph.Ref tags map[string]graph.Ref } func (it *iteratorBase) TagResults(m map[string]graph.Ref) { for tag, val := range it.tags { m[tag] = val } } func (it *iteratorBase) Result() graph.Ref { return it.res } func (it *iteratorBase) ensureColumns() { if it.cols != nil { return } it.cols = it.query.Columns() it.cind = make(map[quad.Direction]int, len(quad.Directions)+1) for i, name := range it.cols { if !strings.HasPrefix(name, tagPref) { continue } if name == tagNode { it.cind[quad.Any] = i continue } name = name[len(tagPref):] for _, d := range quad.Directions { if name == d.String() { it.cind[d] = i break } } } } func (it *iteratorBase) scanValue(r *sql.Rows) bool { it.ensureColumns() nodes := make([]NodeHash, len(it.cols)) pointers := make([]interface{}, len(nodes)) for i := range pointers { pointers[i] = &nodes[i] } if err := r.Scan(pointers...); err != nil { it.err = err return false } it.tags = make(map[string]graph.Ref) for i, name := range it.cols { if !strings.Contains(name, tagPref) { it.tags[name] = nodes[i].ValueHash } } if len(it.cind) > 1 { var q QuadHashes for _, d := range quad.Directions { i, ok := it.cind[d] if !ok { it.err = fmt.Errorf("cannot find quad %v in query output (columns: %v)", d, it.cols) return false } q.Set(d, nodes[i].ValueHash) } it.res = q return true } i, ok := it.cind[quad.Any] if !ok { it.err = fmt.Errorf("cannot find node hash in query output (columns: %v, cind: %v)", it.cols, it.cind) return false } it.res = nodes[i] return true } func (it *iteratorBase) Err() error { return it.err } func (it *iteratorBase) String() string { return it.query.SQL(NewBuilder(it.qs.flavor.QueryDialect)) } func (qs *QuadStore) newIteratorNext(s Select) *iteratorNext { return &iteratorNext{ iteratorBase: newIteratorBase(qs, s), } } type iteratorNext struct { iteratorBase cursor *sql.Rows // TODO(dennwc): nextPath workaround; remove when we get rid of NextPath in general nextPathRes graph.Ref nextPathTags map[string]graph.Ref } func (it *iteratorNext) Next(ctx context.Context) bool { if it.err != nil { return false } if it.cursor == nil { it.cursor, it.err = it.qs.Query(ctx, it.query) } // TODO(dennwc): this loop exists only because of nextPath workaround for { if it.err != nil { return false } if it.nextPathRes != nil { it.res = it.nextPathRes it.tags = it.nextPathTags it.nextPathRes = nil it.nextPathTags = nil return true } if !it.cursor.Next() { it.err = it.cursor.Err() it.cursor.Close() return false } prev := it.res if !it.scanValue(it.cursor) { return false } if !it.query.nextPath { return true } if prev == nil || prev.Key() != it.res.Key() { return true } // skip the same main key if in nextPath mode // the user should receive accept those results via NextPath of the iterator } } func (it *iteratorNext) NextPath(ctx context.Context) bool { if it.err != nil { return false } if !it.query.nextPath { return false } if !it.cursor.Next() { it.err = it.cursor.Err() it.cursor.Close() return false } prev := it.res if !it.scanValue(it.cursor) { return false } if prev.Key() == it.res.Key() { return true } // different main keys - return false, but keep this results for the Next it.nextPathRes = it.res it.nextPathTags = it.tags it.res = nil it.tags = nil return false } func (it *iteratorNext) Close() error { if it.cursor != nil { it.cursor.Close() it.cursor = nil } return nil } func (qs *QuadStore) newIteratorContains(s Select) *iteratorContains { return &iteratorContains{ iteratorBase: newIteratorBase(qs, s), } } type iteratorContains struct { iteratorBase // TODO(dennwc): nextPath workaround; remove when we get rid of NextPath in general nextPathRows *sql.Rows } func (it *iteratorContains) Contains(ctx context.Context, v graph.Ref) bool { it.ensureColumns() sel := it.query sel.Where = append([]Where{}, sel.Where...) switch v := v.(type) { case NodeHash: i, ok := it.cind[quad.Any] if !ok { return false } f := it.query.Fields[i] sel.WhereEq(f.Table, f.Name, v) case QuadHashes: for _, d := range quad.Directions { i, ok := it.cind[d] if !ok { return false } h := v.Get(d) if !h.Valid() { continue } f := it.query.Fields[i] sel.WhereEq(f.Table, f.Name, NodeHash{h}) } default: return false } rows, err := it.qs.Query(ctx, sel) if err != nil { it.err = err return false } if it.query.nextPath { if it.nextPathRows != nil { _ = it.nextPathRows.Close() } it.nextPathRows = rows } else { defer rows.Close() } if !rows.Next() { it.err = rows.Err() return false } return it.scanValue(rows) } func (it *iteratorContains) NextPath(ctx context.Context) bool { if it.err != nil { return false } if !it.query.nextPath { return false } if !it.nextPathRows.Next() { it.err = it.nextPathRows.Err() return false } return it.scanValue(it.nextPathRows) } func (it *iteratorContains) Close() error { if it.nextPathRows != nil { return it.nextPathRows.Close() } return nil } ================================================ FILE: graph/sql/mysql/mysql.go ================================================ package mysql import ( "database/sql" "fmt" "strings" "github.com/cayleygraph/quad" "github.com/go-sql-driver/mysql" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" graphlog "github.com/cayleygraph/cayley/graph/log" csql "github.com/cayleygraph/cayley/graph/sql" ) const Type = "mysql" var QueryDialect = csql.QueryDialect{ RegexpOp: "REGEXP", FieldQuote: func(name string) string { return "`" + name + "`" }, Placeholder: func(n int) string { return "?" }, } func init() { csql.Register(Type, csql.Registration{ Driver: "mysql", HashType: fmt.Sprintf(`BINARY(%d)`, quad.HashSize), BytesType: `BLOB`, HorizonType: `SERIAL`, TimeType: `DATETIME(6)`, QueryDialect: QueryDialect, NoOffsetWithoutLimit: true, CustomNullTime: true, Error: func(err error) error { return err }, Estimated: nil, RunTx: runTxMysql, }) } func runTxMysql(tx *sql.Tx, nodes []graphlog.NodeUpdate, quads []graphlog.QuadUpdate, opts graph.IgnoreOpts) error { // update node ref counts and insert nodes var ( // prepared statements for each value type insertValue = make(map[csql.ValueType]*sql.Stmt) updateValue *sql.Stmt ) for _, n := range nodes { if n.RefInc >= 0 { nodeKey, values, err := csql.NodeValues(csql.NodeHash{ValueHash: n.Hash}, n.Val) if err != nil { return err } values = append([]interface{}{n.RefInc}, values...) values = append(values, n.RefInc) // one more time for UPDATE stmt, ok := insertValue[nodeKey] if !ok { var ph = make([]string, len(values)-1) // excluding last increment for i := range ph { ph[i] = "?" } stmt, err = tx.Prepare(`INSERT INTO nodes(refs, hash, ` + strings.Join(nodeKey.Columns(), ", ") + `) VALUES (` + strings.Join(ph, ", ") + `) ON DUPLICATE KEY UPDATE refs = refs + ?;`) if err != nil { return err } insertValue[nodeKey] = stmt } _, err = stmt.Exec(values...) err = convInsertError(err) if err != nil { clog.Errorf("couldn't exec INSERT statement: %v", err) return err } } else { panic("unexpected node update") } } for _, s := range insertValue { s.Close() } if s := updateValue; s != nil { s.Close() } insertValue = nil updateValue = nil // now we can deal with quads ignore := "" if opts.IgnoreDup { ignore = " IGNORE" } var ( insertQuad *sql.Stmt err error ) for _, d := range quads { dirs := make([]interface{}, 0, len(quad.Directions)) for _, h := range d.Quad.Dirs() { dirs = append(dirs, csql.NodeHash{ValueHash: h}.SQLValue()) } if !d.Del { if insertQuad == nil { insertQuad, err = tx.Prepare(`INSERT` + ignore + ` INTO quads(subject_hash, predicate_hash, object_hash, label_hash, ts) VALUES (?, ?, ?, ?, now());`) if err != nil { return err } insertValue = make(map[csql.ValueType]*sql.Stmt) } _, err := insertQuad.Exec(dirs...) err = convInsertError(err) if err != nil { if _, ok := err.(*graph.DeltaError); !ok { clog.Errorf("couldn't exec INSERT statement: %v", err) } return err } } else { panic("unexpected quad delete") } } return nil } func convInsertError(err error) error { if err == nil { return nil } if e, ok := err.(*mysql.MySQLError); ok { if e.Number == 1062 { // TODO: reference to delta return &graph.DeltaError{Err: graph.ErrQuadExists} } } return err } ================================================ FILE: graph/sql/mysql/mysql_test.go ================================================ //go:build docker // +build docker package mysql import ( "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/sql/sqltest" "github.com/cayleygraph/cayley/internal/dock" ) func makeMysqlVersion(image string) sqltest.DatabaseFunc { return func(t testing.TB) (string, graph.Options) { var conf dock.Config conf.Image = image conf.Tty = true conf.Env = []string{ `MYSQL_ROOT_PASSWORD=root`, `MYSQL_DATABASE=testdb`, } addr := dock.RunAndWait(t, conf, "3306", nil) addr = `root:root@tcp(` + addr + `)/testdb` return addr, nil } } const ( mysqlImage = "mysql:8" mariadbImage = "mariadb:11" ) func TestMysql(t *testing.T) { sqltest.TestAll(t, Type, makeMysqlVersion(mysqlImage), nil) } func TestMariaDB(t *testing.T) { sqltest.TestAll(t, Type, makeMysqlVersion(mariadbImage), nil) } func BenchmarkMysql(t *testing.B) { sqltest.BenchmarkAll(t, Type, makeMysqlVersion(mysqlImage), nil) } func BenchmarkMariadb(t *testing.B) { sqltest.BenchmarkAll(t, Type, makeMysqlVersion(mariadbImage), nil) } ================================================ FILE: graph/sql/optimizer.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package sql import ( "context" "fmt" "sort" "strings" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) func NewOptimizer() *Optimizer { return &Optimizer{} } type Optimizer struct { tableInd int regexpOp CmpOp noOffsetWithoutLimit bool // blame mysql } func (opt *Optimizer) SetRegexpOp(op CmpOp) { opt.regexpOp = op } func (opt *Optimizer) NoOffsetWithoutLimit() { opt.noOffsetWithoutLimit = true } func (opt *Optimizer) nextTable() string { opt.tableInd++ return fmt.Sprintf("t_%d", opt.tableInd) } func (opt *Optimizer) ensureAliases(s *Select) { for i, src := range s.From { if t, ok := src.(Table); ok && t.Alias == "" { t.Alias = opt.nextTable() s.From[i] = t // TODO: copy slice for j := range s.Fields { f := &s.Fields[j] if f.Table == "" { f.Table = t.Alias } } for j := range s.Where { w := &s.Where[j] if w.Table == "" { w.Table = t.Alias } } } } } func sortDirs(dirs []quad.Direction) { sort.Slice(dirs, func(i, j int) bool { return dirs[i] < dirs[j] }) } func (opt *Optimizer) OptimizeShape(ctx context.Context, s shape.Shape) (shape.Shape, bool) { switch s := s.(type) { case shape.AllNodes: return AllNodes(), true case shape.Lookup: return opt.optimizeLookup(s) case shape.Filter: return opt.optimizeFilters(s) case shape.Intersect: return opt.optimizeIntersect(s) case shape.Quads: return opt.optimizeQuads(s) case shape.NodesFrom: return opt.optimizeNodesFrom(s) case shape.QuadsAction: return opt.optimizeQuadsAction(s) case shape.Save: return opt.optimizeSave(s) case shape.Page: return opt.optimizePage(s) default: return s, false } } func selectValueQuery(v quad.Value, op CmpOp) ([]Where, []Value, bool) { if op == OpEqual { // we can use hash to check equality return []Where{ {Field: "hash", Op: op, Value: Placeholder{}}, }, []Value{ HashOf(v), }, true } var ( where []Where params []Value ) switch v := v.(type) { case quad.IRI: where = []Where{ {Field: "value_string", Op: op, Value: Placeholder{}}, {Field: "iri", Op: OpIsTrue}, } params = []Value{ StringVal(v), } case quad.BNode: where = []Where{ {Field: "value_string", Op: op, Value: Placeholder{}}, {Field: "bnode", Op: OpIsTrue}, } params = []Value{ StringVal(v), } case quad.String: where = []Where{ {Field: "value_string", Op: op, Value: Placeholder{}}, {Field: "iri", Op: OpIsNull}, {Field: "bnode", Op: OpIsNull}, {Field: "datatype", Op: OpIsNull}, {Field: "language", Op: OpIsNull}, } params = []Value{ StringVal(v), } case quad.LangString: where = []Where{ {Field: "value_string", Op: op, Value: Placeholder{}}, {Field: "language", Op: OpEqual, Value: Placeholder{}}, } params = []Value{ StringVal(v.Value), StringVal(v.Lang), } case quad.TypedString: where = []Where{ {Field: "value_string", Op: op, Value: Placeholder{}}, {Field: "datatype", Op: OpEqual, Value: Placeholder{}}, } params = []Value{ StringVal(v.Value), StringVal(v.Type), } case quad.Int: where = []Where{ {Field: "value_int", Op: op, Value: Placeholder{}}, } params = []Value{ IntVal(v), } case quad.Float: where = []Where{ {Field: "value_float", Op: op, Value: Placeholder{}}, } params = []Value{ FloatVal(v), } case quad.Bool: where = []Where{ {Field: "value_bool", Op: op, Value: Placeholder{}}, } params = []Value{ BoolVal(v), } case quad.Time: where = []Where{ {Field: "value_time", Op: op, Value: Placeholder{}}, } params = []Value{ TimeVal(v), } default: return nil, nil, false } return where, params, true } func SelectValue(v quad.Value, op CmpOp) *Select { where, params, ok := selectValueQuery(v, op) if !ok { return nil } sel := Nodes(where, params) return &sel } func (opt *Optimizer) optimizeLookup(s shape.Lookup) (shape.Shape, bool) { if len(s) != 1 { // TODO: support for IN return s, false } sel := SelectValue(s[0], OpEqual) if sel == nil { return s, false } return *sel, true } func convRegexp(re string) string { return re // TODO: convert regular expression } func (opt *Optimizer) optimizeFilter(from shape.Shape, f shape.ValueFilter) ([]Where, []Value, bool) { switch f := f.(type) { case shape.Comparison: var cmp CmpOp switch f.Op { case iterator.CompareGT: cmp = OpGT case iterator.CompareGTE: cmp = OpGTE case iterator.CompareLT: cmp = OpLT case iterator.CompareLTE: cmp = OpLTE default: return nil, nil, false } return selectValueQuery(f.Val, cmp) case shape.Wildcard: if opt.regexpOp == "" { return nil, nil, false } return []Where{ {Field: "value_string", Op: opt.regexpOp, Value: Placeholder{}}, }, []Value{ StringVal(convRegexp(f.Regexp())), }, true case shape.Regexp: if opt.regexpOp == "" { return nil, nil, false } where := []Where{ {Field: "value_string", Op: opt.regexpOp, Value: Placeholder{}}, } if !f.Refs { where = append(where, []Where{ {Field: "iri", Op: OpIsNull}, {Field: "bnode", Op: OpIsNull}, }...) } return where, []Value{ StringVal(convRegexp(f.Re.String())), }, true default: return nil, nil, false } } func (opt *Optimizer) optimizeFilters(s shape.Filter) (shape.Shape, bool) { switch from := s.From.(type) { case shape.AllNodes: case Select: if !from.isAll() { return s, false } t, ok := from.From[0].(Table) if !ok || t.Name != "nodes" { return s, false } default: return s, false } var ( where []Where params []Value ) left := shape.Filter{ From: s.From, } for _, f := range s.Filters { if w, p, ok := opt.optimizeFilter(s.From, f); ok { where = append(where, w...) params = append(params, p...) } else { left.Filters = append(left.Filters, f) } } if len(where) == 0 { return s, false } sel := Nodes(where, params) if len(left.Filters) == 0 { return sel, true } left.From = sel return left, true } func (opt *Optimizer) optimizeQuads(s shape.Quads) (shape.Shape, bool) { t1 := opt.nextTable() sel := AllQuads(t1) for _, f := range s { wr := Where{ Table: t1, Field: dirField(f.Dir), Op: OpEqual, } switch fv := f.Values.(type) { case shape.Fixed: if len(fv) != 1 { // TODO: support IN, or generate SELECT equivalent return s, false } wr.Value = sel.AppendParam(fv[0].(Value)) sel.Where = append(sel.Where, wr) case Select: if len(fv.Fields) == 1 { // simple case - just add subquery to FROM tbl := opt.nextTable() sel.From = append(sel.From, Subquery{ Query: fv, Alias: tbl, }) wr.Value = FieldName{ Name: fv.Fields[0].NameOrAlias(), Table: tbl, } sel.Where = append(sel.Where, wr) continue } else if fv.onlyAsSubquery() { // TODO: generic subquery: pass all tags to main query, set WHERE on specific direction, drop __* tags return s, false } opt.ensureAliases(&fv) // add all tables from subquery to the main one, but skip __node field - we should add it to WHERE var head Field for _, f := range fv.Fields { if f.Alias == tagNode { for _, w := range fv.Where { if w.Table == f.Table && w.Field == f.Alias { // TODO: if __node was used in WHERE of subquery, we should rewrite it return s, false } } f.Alias = "" head = f continue } sel.Fields = append(sel.Fields, f) } if head.Table == "" { // something is wrong return s, false } sel.From = append(sel.From, fv.From...) sel.Where = append(sel.Where, fv.Where...) sel.Params = append(sel.Params, fv.Params...) wr.Value = FieldName{ Name: head.Name, Table: head.Table, } sel.Where = append(sel.Where, wr) default: return s, false } } return sel, true } func (opt *Optimizer) optimizeNodesFrom(s shape.NodesFrom) (shape.Shape, bool) { sel, ok := s.Quads.(Select) if !ok { return s, false } sel.Fields = append([]Field{}, sel.Fields...) // all we need is to remove all quad-related tags and preserve one with matching direction dir := dirTag(s.Dir) found := false for i := 0; i < len(sel.Fields); i++ { f := &sel.Fields[i] if f.Alias == dir { f.Alias = tagNode found = true } else if strings.HasPrefix(f.Alias, tagPref) { sel.Fields = append(sel.Fields[:i], sel.Fields[i+1:]...) i-- } } if !found { return s, false } // NodesFrom implies that the iterator will use NextPath sel.nextPath = true return sel, true } func (opt *Optimizer) optimizeQuadsAction(s shape.QuadsAction) (shape.Shape, bool) { sel := Select{ Fields: []Field{ {Name: dirField(s.Result), Alias: tagNode}, }, From: []Source{ Table{Name: "quads"}, }, // NodesFrom (that is a part of QuadsAction) implies that the iterator will use NextPath nextPath: true, } var dirs []quad.Direction for d := range s.Save { dirs = append(dirs, d) } sortDirs(dirs) for _, d := range dirs { for _, t := range s.Save[d] { sel.Fields = append(sel.Fields, Field{ Name: dirField(d), Alias: t, }) } } dirs = nil for d := range s.Filter { dirs = append(dirs, d) } sortDirs(dirs) for _, d := range dirs { v := s.Filter[d] sel.WhereEq("", dirField(d), v.(Value)) } return sel, true } func (opt *Optimizer) optimizeSave(s shape.Save) (shape.Shape, bool) { sel, ok := s.From.(Select) if !ok { return s, false } // find primary value used by iterators fi := -1 for i, f := range sel.Fields { if f.Alias == tagNode { fi = i break } } if fi < 0 { return s, false } // add SELECT fields as aliases for primary field f := sel.Fields[fi] fields := make([]Field, 0, len(s.Tags)+len(sel.Fields)) for _, tag := range s.Tags { f.Alias = tag fields = append(fields, f) } // add other fields fields = append(fields, sel.Fields...) sel.Fields = fields return sel, true } func (opt *Optimizer) optimizePage(s shape.Page) (shape.Shape, bool) { sel, ok := s.From.(Select) if !ok { return s, false } // do not optimize if db only can use offset with limit, and we have no limits set if opt.noOffsetWithoutLimit && sel.Limit == 0 && s.Limit == 0 { return s, false } // call shapes optimizer to calculate correct skip and limit p := shape.Page{ Skip: sel.Offset, Limit: sel.Limit, }.ApplyPage(s) if p == nil { // no intersection - no results return nil, true } sel.Limit = p.Limit sel.Offset = p.Skip return sel, true } func (opt *Optimizer) optimizeIntersect(s shape.Intersect) (shape.Shape, bool) { var ( sels []Select other shape.Intersect ) // we will add our merged Select to this slot other = append(other, nil) for _, sub := range s { // TODO: sort by onlySubquery flag first if sel, ok := sub.(Select); ok && !sel.onlyAsSubquery() { sels = append(sels, sel) } else { other = append(other, sub) } } if len(sels) <= 1 { return s, false } for i := range sels { sels[i] = sels[i].Clone() opt.ensureAliases(&sels[i]) } pri := sels[0] var head *Field for i, f := range pri.Fields { if f.Alias == tagNode { head = &pri.Fields[i] break } } if head == nil { return s, false } sec := sels[1:] nextPath := false for _, s2 := range sec { // merge From, Where and Params pri.From = append(pri.From, s2.From...) pri.Where = append(pri.Where, s2.Where...) pri.Params = append(pri.Params, s2.Params...) nextPath = nextPath || s2.nextPath // also find and remove primary tag, but add the same field to WHERE ok := false for _, f := range s2.Fields { if f.Alias == tagNode { ok = true pri.Where = append(pri.Where, Where{ Table: head.Table, Field: head.Name, Op: OpEqual, Value: FieldName{ Table: f.Table, Name: f.Name, }, }) } else { pri.Fields = append(pri.Fields, f) } } if !ok { return s, false } } if len(other) == 1 { pri.nextPath = pri.nextPath || nextPath return pri, true } other[0] = pri return other, true } ================================================ FILE: graph/sql/postgres/postgres.go ================================================ package postgres import ( "database/sql" "fmt" "strconv" "strings" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/log" csql "github.com/cayleygraph/cayley/graph/sql" "github.com/cayleygraph/quad" "github.com/lib/pq" ) const Type = "postgres" var QueryDialect = csql.QueryDialect{ RegexpOp: "~", FieldQuote: pq.QuoteIdentifier, Placeholder: func(n int) string { return fmt.Sprintf("$%d", n) }, } func init() { csql.Register(Type, csql.Registration{ Driver: "postgres", HashType: `BYTEA`, BytesType: `BYTEA`, HorizonType: `BIGSERIAL`, TimeType: `timestamp with time zone`, QueryDialect: QueryDialect, ConditionalIndexes: true, FillFactor: true, Error: ConvError, Estimated: func(table string) string { return "SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='" + table + "';" }, RunTx: RunTxPostgres, }) } func ConvError(err error) error { e, ok := err.(*pq.Error) if !ok { return err } switch e.Code { case "42P07": return graph.ErrDatabaseExists } return err } func convInsertError(err error) error { if err == nil { return err } if pe, ok := err.(*pq.Error); ok { if pe.Code == "23505" { // TODO: reference to delta return &graph.DeltaError{Err: graph.ErrQuadExists} } } return err } //func copyFromPG(tx *sql.Tx, in []graph.Delta, opts graph.IgnoreOpts) error { // panic("broken") // stmt, err := tx.Prepare(pq.CopyIn("quads", "subject", "predicate", "object", "label", "id", "ts", "subject_hash", "predicate_hash", "object_hash", "label_hash")) // if err != nil { // clog.Errorf("couldn't prepare COPY statement: %v", err) // return err // } // for _, d := range in { // s, p, o, l, err := marshalQuadDirections(d.Quad) // if err != nil { // clog.Errorf("couldn't marshal quads: %v", err) // return err // } // _, err = stmt.Exec( // s, // p, // o, // l, // d.ID.Int(), // d.Timestamp, // hashOf(d.Quad.Subject), // hashOf(d.Quad.Predicate), // hashOf(d.Quad.Object), // hashOf(d.Quad.Label), // ) // if err != nil { // err = convInsertErrorPG(err) // clog.Errorf("couldn't execute COPY statement: %v", err) // return err // } // } // _, err = stmt.Exec() // if err != nil { // err = convInsertErrorPG(err) // return err // } // _ = stmt.Close() // COPY will be closed on last Exec, this will return non-nil error in all cases // return nil //} func RunTxPostgres(tx *sql.Tx, nodes []graphlog.NodeUpdate, quads []graphlog.QuadUpdate, opts graph.IgnoreOpts) error { return RunTx(tx, nodes, quads, opts, "") } func RunTx(tx *sql.Tx, nodes []graphlog.NodeUpdate, quads []graphlog.QuadUpdate, opts graph.IgnoreOpts, onConflict string) error { // update node ref counts and insert nodes var ( // prepared statements for each value type insertValue = make(map[csql.ValueType]*sql.Stmt) updateValue *sql.Stmt ) for _, n := range nodes { if n.RefInc >= 0 { nodeKey, values, err := csql.NodeValues(csql.NodeHash{ValueHash: n.Hash}, n.Val) if err != nil { return err } values = append([]interface{}{n.RefInc}, values...) stmt, ok := insertValue[nodeKey] if !ok { var ph = make([]string, len(values)) for i := range ph { ph[i] = "$" + strconv.FormatInt(int64(i)+1, 10) } stmt, err = tx.Prepare(`INSERT INTO nodes(refs, hash, ` + strings.Join(nodeKey.Columns(), ", ") + `) VALUES (` + strings.Join(ph, ", ") + `) ON CONFLICT (hash) DO UPDATE SET refs = nodes.refs + EXCLUDED.refs;`) if err != nil { return err } insertValue[nodeKey] = stmt } _, err = stmt.Exec(values...) err = convInsertError(err) if err != nil { clog.Errorf("couldn't exec INSERT statement: %v", err) return err } } else { panic("unexpected node update") } } for _, s := range insertValue { s.Close() } if s := updateValue; s != nil { s.Close() } insertValue = nil updateValue = nil // now we can deal with quads // TODO: copy //if allAdds && !opts.IgnoreDup { // return qs.copyFrom(tx, in, opts) //} end := ";" if opts.IgnoreDup { end = ` ON CONFLICT ` + onConflict + ` DO NOTHING;` } var ( insertQuad *sql.Stmt err error ) for _, d := range quads { dirs := make([]interface{}, 0, len(quad.Directions)) for _, h := range d.Quad.Dirs() { dirs = append(dirs, csql.NodeHash{ValueHash: h}.SQLValue()) } if !d.Del { if insertQuad == nil { insertQuad, err = tx.Prepare(`INSERT INTO quads(subject_hash, predicate_hash, object_hash, label_hash, ts) VALUES ($1, $2, $3, $4, now())` + end) if err != nil { return err } insertValue = make(map[csql.ValueType]*sql.Stmt) } _, err := insertQuad.Exec(dirs...) err = convInsertError(err) if err != nil { if _, ok := err.(*graph.DeltaError); !ok { clog.Errorf("couldn't exec INSERT statement: %v", err) } return err } } else { panic("unexpected quad delete") } } return nil } ================================================ FILE: graph/sql/postgres/postgres_test.go ================================================ //go:build docker // +build docker package postgres import ( "testing" "github.com/lib/pq" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/sql/sqltest" "github.com/cayleygraph/cayley/internal/dock" ) func makePostgres(t testing.TB) (string, graph.Options) { var conf dock.Config conf.Image = "postgres:16" conf.OpenStdin = true conf.Tty = true conf.Env = []string{`POSTGRES_PASSWORD=postgres`} addr := dock.RunAndWait(t, conf, "5432", func(addr string) bool { conn, err := pq.Open(`postgres://postgres:postgres@` + addr + `/postgres?sslmode=disable`) if err != nil { return false } conn.Close() return true }) addr = `postgres://postgres:postgres@` + addr + `/postgres?sslmode=disable` return addr, nil } var conf = &sqltest.Config{ TimeRound: true, TimeInMcs: true, } func TestPostgres(t *testing.T) { sqltest.TestAll(t, Type, makePostgres, conf) } func BenchmarkPostgres(t *testing.B) { sqltest.BenchmarkAll(t, Type, makePostgres, conf) } ================================================ FILE: graph/sql/quadstore.go ================================================ package sql import ( "context" "database/sql" "database/sql/driver" "fmt" "strings" "sync" "time" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/pquads" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" graphlog "github.com/cayleygraph/cayley/graph/log" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/internal/lru" ) func registerQuadStore(name, typ string) { graph.RegisterQuadStore(name, graph.QuadStoreRegistration{ NewFunc: func(addr string, options graph.Options) (graph.QuadStore, error) { return New(typ, addr, options) }, UpgradeFunc: nil, InitFunc: func(addr string, options graph.Options) error { return Init(typ, addr, options) }, IsPersistent: true, }) } var _ Value = StringVal("") type StringVal string func (v StringVal) SQLValue() interface{} { return escapeNullByte(string(v)) } type IntVal int64 func (v IntVal) SQLValue() interface{} { return int64(v) } type FloatVal float64 func (v FloatVal) SQLValue() interface{} { return float64(v) } type BoolVal bool func (v BoolVal) SQLValue() interface{} { return bool(v) } type TimeVal time.Time func (v TimeVal) SQLValue() interface{} { return time.Time(v) } type NodeHash struct { refs.ValueHash } func (h NodeHash) SQLValue() interface{} { if !h.Valid() { return nil } return []byte(h.ValueHash[:]) } func (h *NodeHash) Scan(src interface{}) error { if src == nil { *h = NodeHash{} return nil } b, ok := src.([]byte) if !ok { return fmt.Errorf("cannot scan %T to NodeHash", src) } if len(b) == 0 { *h = NodeHash{} return nil } else if len(b) != quad.HashSize { return fmt.Errorf("unexpected hash length: %d", len(b)) } copy(h.ValueHash[:], b) return nil } func HashOf(s quad.Value) NodeHash { return NodeHash{refs.HashOf(s)} } type QuadHashes struct { refs.QuadHash } type QuadStore struct { db *sql.DB opt *Optimizer flavor Registration ids *lru.Cache sizes *lru.Cache noSizes bool mu sync.RWMutex nodes int64 quads int64 } func connect(addr string, flavor string, opts graph.Options) (*sql.DB, error) { maxOpenConnections, err := opts.IntKey("maxopenconnections", -1) if err != nil { return nil, fmt.Errorf("could not retrieve maxopenconnections from options: %v", err) } maxIdleConnections, err := opts.IntKey("maxidleconnections", -1) if err != nil { return nil, fmt.Errorf("could not retrieve maxIdleConnections from options: %v", err) } connMaxLifetime, err := opts.StringKey("connmaxlifetime", "") if err != nil { return nil, fmt.Errorf("could not retrieve connmaxlifetime from options: %v", err) } var connDuration time.Duration if connMaxLifetime != "" { connDuration, err = time.ParseDuration(connMaxLifetime) if err != nil { return nil, fmt.Errorf("couldn't parse connmaxlifetime string: %v", err) } } // TODO(barakmich): Parse options for more friendly addr conn, err := sql.Open(flavor, addr) if err != nil { clog.Errorf("Couldn't open database at %s: %#v", addr, err) return nil, err } // "Open may just validate its arguments without creating a connection to the database." // "To verify that the data source name is valid, call Ping." // Source: http://golang.org/pkg/database/sql/#Open if err := conn.Ping(); err != nil { clog.Errorf("Couldn't open database at %s: %#v", addr, err) return nil, err } if maxOpenConnections != -1 { conn.SetMaxOpenConns(maxOpenConnections) } if maxIdleConnections != -1 { conn.SetMaxIdleConns(maxIdleConnections) } if connDuration != 0 { conn.SetConnMaxLifetime(connDuration) } return conn, nil } var nodesColumns = []string{ "hash", "value", "value_string", "datatype", "language", "iri", "bnode", "value_int", "value_bool", "value_float", "value_time", } var nodeInsertColumns = [][]string{ {"value"}, {"value_string", "iri"}, {"value_string", "bnode"}, {"value_string"}, {"value_string", "datatype"}, {"value_string", "language"}, {"value_int"}, {"value_bool"}, {"value_float"}, {"value_time"}, } func Init(typ string, addr string, options graph.Options) error { fl, ok := types[typ] if !ok { return fmt.Errorf("unsupported sql database: %q", typ) } conn, err := connect(addr, fl.Driver, options) if err != nil { return err } defer conn.Close() nodesSQL := fl.nodesTable() quadsSQL := fl.quadsTable() indexes := fl.quadIndexes(options) if fl.NoSchemaChangesInTx { _, err = conn.Exec(nodesSQL) if err != nil { err = fl.Error(err) clog.Errorf("Cannot create nodes table: %v", err) return err } _, err = conn.Exec(quadsSQL) if err != nil { err = fl.Error(err) clog.Errorf("Cannot create quad table: %v", err) return err } for _, index := range indexes { if _, err = conn.Exec(index); err != nil { clog.Errorf("Cannot create index: %v", err) return err } } } else { tx, err := conn.Begin() if err != nil { clog.Errorf("Couldn't begin creation transaction: %s", err) return err } _, err = tx.Exec(nodesSQL) if err != nil { tx.Rollback() err = fl.Error(err) clog.Errorf("Cannot create nodes table: %v", err) return err } _, err = tx.Exec(quadsSQL) if err != nil { tx.Rollback() err = fl.Error(err) clog.Errorf("Cannot create quad table: %v", err) return err } for _, index := range indexes { if _, err = tx.Exec(index); err != nil { clog.Errorf("Cannot create index: %v", err) tx.Rollback() return err } } tx.Commit() } return nil } func New(typ string, addr string, options graph.Options) (graph.QuadStore, error) { fl, ok := types[typ] if !ok { return nil, fmt.Errorf("unsupported sql database: %q", typ) } conn, err := connect(addr, fl.Driver, options) if err != nil { return nil, err } qs := &QuadStore{ db: conn, opt: NewOptimizer(), flavor: fl, quads: -1, nodes: -1, sizes: lru.New(1024), ids: lru.New(1024), noSizes: true, // Skip size checking by default. } qs.opt.SetRegexpOp(qs.flavor.RegexpOp) if qs.flavor.NoOffsetWithoutLimit { qs.opt.NoOffsetWithoutLimit() } if local, err := options.BoolKey("local_optimize", false); err != nil { return nil, err } else if ok && local { qs.noSizes = false } return qs, nil } func escapeNullByte(s string) string { return strings.Replace(s, "\u0000", `\x00`, -1) } func unescapeNullByte(s string) string { return strings.Replace(s, `\x00`, "\u0000", -1) } type ValueType int func (t ValueType) Columns() []string { return nodeInsertColumns[t] } func NodeValues(h NodeHash, v quad.Value) (ValueType, []interface{}, error) { var ( nodeKey ValueType values = []interface{}{h.SQLValue(), nil, nil}[:1] ) switch v := v.(type) { case quad.IRI: nodeKey = 1 values = append(values, string(v), true) case quad.BNode: nodeKey = 2 values = append(values, string(v), true) case quad.String: nodeKey = 3 values = append(values, escapeNullByte(string(v))) case quad.TypedString: nodeKey = 4 values = append(values, escapeNullByte(string(v.Value)), string(v.Type)) case quad.LangString: nodeKey = 5 values = append(values, escapeNullByte(string(v.Value)), v.Lang) case quad.Int: nodeKey = 6 values = append(values, int64(v)) case quad.Bool: nodeKey = 7 values = append(values, bool(v)) case quad.Float: nodeKey = 8 values = append(values, float64(v)) case quad.Time: nodeKey = 9 values = append(values, time.Time(v)) default: nodeKey = 0 p, err := pquads.MarshalValue(v) if err != nil { clog.Errorf("couldn't marshal value: %v", err) return 0, nil, err } values = append(values, p) } return nodeKey, values, nil } func (qs *QuadStore) NewQuadWriter() (quad.WriteCloser, error) { return &quadWriter{qs: qs}, nil } type quadWriter struct { qs *QuadStore deltas []graph.Delta } func (w *quadWriter) WriteQuad(q quad.Quad) error { _, err := w.WriteQuads([]quad.Quad{q}) return err } func (w *quadWriter) WriteQuads(buf []quad.Quad) (int, error) { // TODO(dennwc): write an optimized implementation w.deltas = w.deltas[:0] if cap(w.deltas) < len(buf) { w.deltas = make([]graph.Delta, 0, len(buf)) } for _, q := range buf { w.deltas = append(w.deltas, graph.Delta{ Quad: q, Action: graph.Add, }) } err := w.qs.ApplyDeltas(w.deltas, graph.IgnoreOpts{ IgnoreDup: true, }) w.deltas = w.deltas[:0] if err != nil { return 0, err } return len(buf), nil } func (w *quadWriter) Close() error { w.deltas = nil return nil } func (qs *QuadStore) ApplyDeltas(in []graph.Delta, opts graph.IgnoreOpts) error { // first calculate values ref deltas deltas := graphlog.SplitDeltas(in) tx, err := qs.db.Begin() if err != nil { clog.Errorf("couldn't begin write transaction: %v", err) return err } retry := qs.flavor.TxRetry if retry == nil { retry = func(tx *sql.Tx, stmts func() error) error { return stmts() } } p := make([]string, 4) for i := range p { p[i] = qs.flavor.Placeholder(i + 1) } err = retry(tx, func() error { err = qs.flavor.RunTx(tx, deltas.IncNode, deltas.QuadAdd, opts) if err != nil { return err } // quad delete is also generic, execute here var ( deleteQuad *sql.Stmt deleteTriple *sql.Stmt ) fixNodes := make(map[refs.ValueHash]int) for _, d := range deltas.QuadDel { dirs := make([]interface{}, 0, len(quad.Directions)) for _, h := range d.Quad.Dirs() { dirs = append(dirs, NodeHash{h}.SQLValue()) } if deleteQuad == nil { deleteQuad, err = tx.Prepare(`DELETE FROM quads WHERE subject_hash=` + p[0] + ` and predicate_hash=` + p[1] + ` and object_hash=` + p[2] + ` and label_hash=` + p[3] + `;`) if err != nil { return err } deleteTriple, err = tx.Prepare(`DELETE FROM quads WHERE subject_hash=` + p[0] + ` and predicate_hash=` + p[1] + ` and object_hash=` + p[2] + ` and label_hash is null;`) if err != nil { return err } } stmt := deleteQuad if i := len(dirs) - 1; dirs[i] == nil { stmt = deleteTriple dirs = dirs[:i] } result, err := stmt.Exec(dirs...) if err != nil { clog.Errorf("couldn't exec DELETE statement: %v", err) return err } affected, err := result.RowsAffected() if err != nil { clog.Errorf("couldn't get DELETE RowsAffected: %v", err) return err } if affected != 1 { if !opts.IgnoreMissing { // TODO: reference to delta return &graph.DeltaError{Err: graph.ErrQuadNotExist} } // revert counters for all directions of this quad for _, dir := range quad.Directions { if h := d.Quad.Get(dir); h.Valid() { fixNodes[h]++ } } } } if len(deltas.DecNode) == 0 { return nil } // node update SQL is generic enough to run it here updateNode, err := tx.Prepare(`UPDATE nodes SET refs = refs + ` + p[0] + ` WHERE hash = ` + p[1] + `;`) if err != nil { return err } for _, n := range deltas.DecNode { n.RefInc += fixNodes[n.Hash] if n.RefInc == 0 { continue } _, err := updateNode.Exec(n.RefInc, NodeHash{n.Hash}.SQLValue()) if err != nil { clog.Errorf("couldn't exec UPDATE statement: %v", err) return err } } // and remove unused nodes at last _, err = tx.Exec(`DELETE FROM nodes WHERE refs <= 0;`) if err != nil { clog.Errorf("couldn't exec DELETE nodes statement: %v", err) return err } return nil }) if err != nil { tx.Rollback() return err } qs.mu.Lock() // TODO(barakmich): Sync size with writes. qs.quads = -1 qs.nodes = -1 qs.mu.Unlock() return tx.Commit() } func (qs *QuadStore) Quad(val graph.Ref) (quad.Quad, error) { h := val.(QuadHashes) var q quad.Quad var err error q.Subject, err = qs.NameOf(h.Get(quad.Subject)) if err != nil { return q, err } q.Predicate, err = qs.NameOf(h.Get(quad.Predicate)) if err != nil { return q, err } q.Object, err = qs.NameOf(h.Get(quad.Object)) if err != nil { return q, err } q.Label, err = qs.NameOf(h.Get(quad.Label)) return q, err } func (qs *QuadStore) QuadIterator(d quad.Direction, val graph.Ref) iterator.Shape { v, ok := val.(Value) if !ok { return iterator.NewNull() } sel := AllQuads("") sel.WhereEq("", dirField(d), v) return qs.newIterator(sel) } func (qs *QuadStore) querySize(ctx context.Context, sel Select) (refs.Size, error) { sel.Fields = []Field{ {Name: "COUNT(*)", Raw: true}, // TODO: proper support for expressions } var sz int64 err := qs.QueryRow(ctx, sel).Scan(&sz) if err != nil { return refs.Size{}, err } return refs.Size{ Value: sz, Exact: true, }, nil } func (qs *QuadStore) QuadIteratorSize(ctx context.Context, d quad.Direction, val graph.Ref) (refs.Size, error) { v, ok := val.(Value) if !ok { return refs.Size{Value: 0, Exact: true}, nil } sel := AllQuads("") sel.WhereEq("", dirField(d), v) return qs.querySize(ctx, sel) } func (qs *QuadStore) NodesAllIterator() iterator.Shape { return qs.newIterator(AllNodes()) } func (qs *QuadStore) QuadsAllIterator() iterator.Shape { return qs.newIterator(AllQuads("")) } func (qs *QuadStore) ValueOf(s quad.Value) (graph.Ref, error) { return NodeHash(HashOf(s)), nil } // NullTime represents a time.Time that may be null. NullTime implements the // sql.Scanner interface so it can be used as a scan destination, similar to // sql.NullString. type NullTime struct { Time time.Time Valid bool // Valid is true if Time is not NULL } // Scan implements the Scanner interface. func (nt *NullTime) Scan(value interface{}) error { if value == nil { nt.Time, nt.Valid = time.Time{}, false return nil } switch value := value.(type) { case time.Time: nt.Time, nt.Valid = value, true case []byte: t, err := time.Parse("2006-01-02 15:04:05.999999", string(value)) if err != nil { return err } nt.Time, nt.Valid = t, true default: return fmt.Errorf("unsupported time format: %T: %v", value, value) } return nil } // Value implements the driver Valuer interface. func (nt NullTime) Value() (driver.Value, error) { if !nt.Valid { return nil, nil } return nt.Time, nil } func (qs *QuadStore) NameOf(v graph.Ref) (quad.Value, error) { if v == nil { return nil, nil } else if v, ok := v.(refs.PreFetchedValue); ok { return v.NameOf(), nil } var hash NodeHash switch h := v.(type) { case refs.PreFetchedValue: return h.NameOf(), nil case NodeHash: hash = h case refs.ValueHash: hash = NodeHash{h} default: return nil, fmt.Errorf("unexpected token: %T", v) } if !hash.Valid() { return nil, nil } if val, ok := qs.ids.Get(hash.String()); ok { return val.(quad.Value), nil } query := `SELECT value, value_string, datatype, language, iri, bnode, value_int, value_bool, value_float, value_time FROM nodes WHERE hash = ` + qs.flavor.Placeholder(1) + ` LIMIT 1;` c := qs.db.QueryRow(query, hash.SQLValue()) var ( data []byte str sql.NullString typ sql.NullString lang sql.NullString iri sql.NullBool bnode sql.NullBool vint sql.NullInt64 vbool sql.NullBool vfloat sql.NullFloat64 vtimeStd sql.NullTime vtimeCustom NullTime ) var ( vtimeScan any vtimeTime *time.Time vtimeValid *bool ) if qs.flavor.CustomNullTime { vtimeScan = &vtimeCustom vtimeTime = &vtimeCustom.Time vtimeValid = &vtimeCustom.Valid } else { vtimeScan = &vtimeStd vtimeTime = &vtimeStd.Time vtimeValid = &vtimeStd.Valid } if err := c.Scan( &data, &str, &typ, &lang, &iri, &bnode, &vint, &vbool, &vfloat, vtimeScan, ); err != nil { if err != sql.ErrNoRows { return nil, fmt.Errorf("error executing value lookup: %w", err) } } var val quad.Value if str.Valid { if iri.Bool { val = quad.IRI(str.String) } else if bnode.Bool { val = quad.BNode(str.String) } else if lang.Valid { val = quad.LangString{ Value: quad.String(unescapeNullByte(str.String)), Lang: lang.String, } } else if typ.Valid { val = quad.TypedString{ Value: quad.String(unescapeNullByte(str.String)), Type: quad.IRI(typ.String), } } else { val = quad.String(unescapeNullByte(str.String)) } } else if vint.Valid { val = quad.Int(vint.Int64) } else if vbool.Valid { val = quad.Bool(vbool.Bool) } else if vfloat.Valid { val = quad.Float(vfloat.Float64) } else if *vtimeValid { val = quad.Time(vtimeTime.UTC()) } else { qv, err := pquads.UnmarshalValue(data) if err != nil { return nil, fmt.Errorf("unmarshal value: %w", err) } val = qv } if val != nil { qs.ids.Put(hash.String(), val) } return val, nil } func (qs *QuadStore) Stats(ctx context.Context, exact bool) (graph.Stats, error) { st := graph.Stats{ Nodes: refs.Size{Exact: true}, Quads: refs.Size{Exact: true}, } qs.mu.RLock() st.Quads.Value = qs.quads st.Nodes.Value = qs.nodes qs.mu.RUnlock() if st.Quads.Value >= 0 { return st, nil } query := func(table string) string { return "SELECT COUNT(*) FROM " + table + ";" } if !exact && qs.flavor.Estimated != nil { query = qs.flavor.Estimated st.Quads.Exact = false st.Nodes.Exact = false } err := qs.db.QueryRow(query("quads")).Scan(&st.Quads.Value) if err != nil { return graph.Stats{}, err } err = qs.db.QueryRow(query("nodes")).Scan(&st.Nodes.Value) if err != nil { return graph.Stats{}, err } if st.Quads.Exact { qs.mu.Lock() qs.quads = st.Quads.Value qs.nodes = st.Nodes.Value qs.mu.Unlock() } return st, nil } func (qs *QuadStore) Close() error { return qs.db.Close() } func (qs *QuadStore) QuadDirection(in graph.Ref, d quad.Direction) (graph.Ref, error) { return NodeHash{in.(QuadHashes).Get(d)}, nil } func (qs *QuadStore) sizeForIterator(dir quad.Direction, hash NodeHash) int64 { var err error if qs.noSizes { st, _ := qs.Stats(context.TODO(), false) if dir == quad.Predicate { return (st.Quads.Value / 100) + 1 } return (st.Quads.Value / 1000) + 1 } if val, ok := qs.sizes.Get(hash.String() + string(dir.Prefix())); ok { return val.(int64) } var size int64 if clog.V(4) { clog.Infof("sql: getting size for select %s, %v", dir.String(), hash) } err = qs.db.QueryRow( fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = "+qs.flavor.Placeholder(1)+";", dir.String()), hash.SQLValue()).Scan(&size) if err != nil { clog.Errorf("Error getting size from SQL database: %v", err) return 0 } qs.sizes.Put(hash.String()+string(dir.Prefix()), size) return size } ================================================ FILE: graph/sql/shape.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package sql import ( "context" "fmt" "strconv" "strings" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) var DefaultDialect = QueryDialect{ FieldQuote: func(s string) string { return strconv.Quote(s) }, Placeholder: func(_ int) string { return "?" }, } type QueryDialect struct { RegexpOp CmpOp FieldQuote func(string) string Placeholder func(int) string } func NewBuilder(d QueryDialect) *Builder { return &Builder{d: d} } type Builder struct { d QueryDialect pi int } func needQuotes(s string) bool { for i, r := range s { if (r < 'a' || r > 'z') && r != '_' && (i == 0 || r < '0' || r > '9') { return true } } return false } func (b *Builder) EscapeField(s string) string { if !needQuotes(s) { return s } return b.d.FieldQuote(s) } func (b *Builder) Placeholder() string { b.pi++ return b.d.Placeholder(b.pi) } const ( tagPref = "__" tagNode = tagPref + "node" ) func dirField(d quad.Direction) string { return d.String() + "_hash" } func dirTag(d quad.Direction) string { return tagPref + d.String() } type Value interface { SQLValue() interface{} } type Shape interface { SQL(b *Builder) string Args() []Value Columns() []string } func AllNodes() Select { return Nodes(nil, nil) } func Nodes(where []Where, params []Value) Select { return Select{ Fields: []Field{ {Name: "hash", Alias: tagNode}, }, From: []Source{ Table{Name: "nodes"}, }, Where: where, Params: params, } } func AllQuads(alias string) Select { sel := Select{ From: []Source{ Table{Name: "quads", Alias: alias}, }, } for _, d := range quad.Directions { sel.Fields = append(sel.Fields, Field{ Table: alias, Name: dirField(d), Alias: dirTag(d), }) } return sel } type FieldName struct { Name string Table string } func (FieldName) isExpr() {} func (f FieldName) SQL(b *Builder) string { name := b.EscapeField(f.Name) if f.Table != "" { name = f.Table + "." + name } return name } type Field struct { Name string Raw bool // do not quote Name Alias string Table string } func (f Field) SQL(b *Builder) string { name := f.Name if !f.Raw { name = b.EscapeField(name) } if f.Table != "" { name = f.Table + "." + name } if f.Alias == "" { return name } return name + " AS " + b.EscapeField(f.Alias) } func (f Field) NameOrAlias() string { if f.Alias != "" { return f.Alias } return f.Name } type Source interface { SQL(b *Builder) string Args() []Value isSource() } type Table struct { Name string Alias string } func (Table) isSource() {} type Subquery struct { Query Select Alias string } func (Subquery) isSource() {} func (s Subquery) SQL(b *Builder) string { q := "(" + s.Query.SQL(b) + ")" if s.Alias != "" { q += " AS " + b.EscapeField(s.Alias) } return q } func (s Subquery) Args() []Value { return s.Query.Args() } func (f Table) SQL(b *Builder) string { if f.Alias == "" { return f.Name } return f.Name + " AS " + b.EscapeField(f.Alias) } func (f Table) Args() []Value { return nil } func (f Table) NameSQL() string { if f.Alias != "" { return f.Alias } return f.Name } type CmpOp string const ( OpEqual = CmpOp("=") OpGT = CmpOp(">") OpGTE = CmpOp(">=") OpLT = CmpOp("<") OpLTE = CmpOp("<=") OpIsNull = CmpOp("IS NULL") OpIsTrue = CmpOp("IS true") ) type Expr interface { isExpr() SQL(b *Builder) string } type Placeholder struct{} func (Placeholder) isExpr() {} func (Placeholder) SQL(b *Builder) string { return b.Placeholder() } type Where struct { Field string Table string Op CmpOp Value Expr } func (w Where) SQL(b *Builder) string { name := w.Field if w.Table != "" { name = w.Table + "." + b.EscapeField(name) } parts := []string{name, string(w.Op)} if w.Value != nil { parts = append(parts, w.Value.SQL(b)) } return strings.Join(parts, " ") } var _ Shape = Select{} // Select is a simplified representation of SQL SELECT query. type Select struct { Fields []Field From []Source Where []Where Params []Value Limit int64 Offset int64 // TODO(dennwc): this field in unexported because we don't want it to a be a part of the API // however, it's necessary to make NodesFrom optimizations to work with SQL nextPath bool } func (s Select) Clone() Select { s.Fields = append([]Field{}, s.Fields...) s.From = append([]Source{}, s.From...) s.Where = append([]Where{}, s.Where...) s.Params = append([]Value{}, s.Params...) return s } func (s Select) isAll() bool { return len(s.From) == 1 && len(s.Where) == 0 && len(s.Params) == 0 && !s.onlyAsSubquery() } // onlyAsSubquery indicates that query cannot be merged into existing SELECT because of some specific properties of query. // An example of such properties might be LIMIT, DISTINCT, etc. func (s Select) onlyAsSubquery() bool { return s.Limit > 0 || s.Offset > 0 } func (s Select) Columns() []string { names := make([]string, 0, len(s.Fields)) for _, f := range s.Fields { name := f.Alias if name == "" { name = f.Name } names = append(names, name) } return names } func (s Select) BuildIterator(qs graph.QuadStore) iterator.Shape { sq, ok := qs.(*QuadStore) if !ok { return iterator.NewError(fmt.Errorf("not a SQL quadstore: %T", qs)) } return sq.newIterator(s) } func (s Select) Optimize(ctx context.Context, r shape.Optimizer) (shape.Shape, bool) { // TODO: call optimize on sub-tables? but what if it decides to de-optimize our SQL shape? return s, false } func (s *Select) AppendParam(o Value) Expr { s.Params = append(s.Params, o) return Placeholder{} } func (s *Select) WhereEq(tbl, field string, v Value) { s.Where = append(s.Where, Where{ Table: tbl, Field: field, Op: OpEqual, Value: s.AppendParam(v), }) } func (s Select) SQL(b *Builder) string { var parts []string var fields []string for _, f := range s.Fields { fields = append(fields, f.SQL(b)) } parts = append(parts, "SELECT "+strings.Join(fields, ", ")) var tables []string for _, t := range s.From { tables = append(tables, t.SQL(b)) } parts = append(parts, "FROM "+strings.Join(tables, ", ")) if len(s.Where) != 0 { var wheres []string for _, w := range s.Where { wheres = append(wheres, w.SQL(b)) } parts = append(parts, "WHERE "+strings.Join(wheres, " AND ")) } if s.Limit > 0 { parts = append(parts, "LIMIT "+strconv.FormatInt(s.Limit, 10)) } if s.Offset > 0 { parts = append(parts, "OFFSET "+strconv.FormatInt(s.Offset, 10)) } sep := " " if len(fields) > 1 { sep = "\n\t" } return strings.Join(parts, sep) } func (s Select) Args() []Value { var args []Value // first add args for FROM subqueries for _, q := range s.From { args = append(args, q.Args()...) } // and add params for WHERE args = append(args, s.Params...) return args } ================================================ FILE: graph/sql/shape_test.go ================================================ package sql import ( "context" "fmt" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" ) type stringVal string func (s stringVal) Key() interface{} { return string(s) } func (s stringVal) SQLValue() interface{} { return string(s) } func sVal(s string) stringVal { return stringVal(s) } func sVals(arr ...string) []Value { out := make([]Value, 0, len(arr)) for _, s := range arr { out = append(out, sVal(s)) } return out } var shapeCases = []struct { skip bool name string s shape.Shape qu string args []Value }{ { name: "all nodes", s: shape.AllNodes{}, qu: `SELECT hash AS ` + tagNode + ` FROM nodes`, }, { name: "lookup iri", s: shape.Lookup{quad.IRI("a")}, qu: `SELECT hash AS ` + tagNode + ` FROM nodes WHERE hash = $1`, args: []Value{HashOf(quad.IRI("a"))}, }, { name: "gt iri", s: shape.Filter{ From: shape.AllNodes{}, Filters: []shape.ValueFilter{ shape.Comparison{Op: iterator.CompareGT, Val: quad.IRI("a")}, }, }, qu: `SELECT hash AS ` + tagNode + ` FROM nodes WHERE value_string > $1 AND iri IS true`, args: []Value{StringVal("a")}, }, { name: "gt string", s: shape.Filter{ From: shape.AllNodes{}, Filters: []shape.ValueFilter{ shape.Comparison{Op: iterator.CompareGT, Val: quad.String("a")}, }, }, qu: `SELECT hash AS ` + tagNode + ` FROM nodes WHERE value_string > $1 AND iri IS NULL AND bnode IS NULL AND datatype IS NULL AND language IS NULL`, args: []Value{StringVal("a")}, }, { name: "gt typed string", s: shape.Filter{ From: shape.AllNodes{}, Filters: []shape.ValueFilter{ shape.Comparison{Op: iterator.CompareGT, Val: quad.TypedString{Value: "a", Type: "A"}}, }, }, qu: `SELECT hash AS ` + tagNode + ` FROM nodes WHERE value_string > $1 AND datatype = $2`, args: []Value{StringVal("a"), StringVal("A")}, }, { name: "lookup int", s: shape.Filter{ From: shape.AllNodes{}, Filters: []shape.ValueFilter{ shape.Comparison{Op: iterator.CompareGT, Val: quad.Int(42)}, }, }, qu: `SELECT hash AS ` + tagNode + ` FROM nodes WHERE value_int > $1`, args: []Value{IntVal(42)}, }, { name: "all quads", s: shape.Quads{}, qu: `SELECT t_1.subject_hash AS __subject, t_1.predicate_hash AS __predicate, t_1.object_hash AS __object, t_1.label_hash AS __label FROM quads AS t_1`, }, { name: "limit quads and skip first", s: shape.Page{From: shape.Quads{}, Limit: 100, Skip: 1}, qu: `SELECT t_1.subject_hash AS __subject, t_1.predicate_hash AS __predicate, t_1.object_hash AS __object, t_1.label_hash AS __label FROM quads AS t_1 LIMIT 100 OFFSET 1`, }, { name: "quads with subject and predicate", s: shape.Quads{ {Dir: quad.Subject, Values: shape.Fixed{sVal("s")}}, {Dir: quad.Predicate, Values: shape.Fixed{sVal("p")}}, }, qu: `SELECT t_1.subject_hash AS __subject, t_1.predicate_hash AS __predicate, t_1.object_hash AS __object, t_1.label_hash AS __label FROM quads AS t_1 WHERE t_1.subject_hash = $1 AND t_1.predicate_hash = $2`, args: sVals("s", "p"), }, { name: "quad actions", s: shape.QuadsAction{ Result: quad.Subject, Save: map[quad.Direction][]string{ quad.Object: {"o1", "o2"}, quad.Label: {"l 1"}, }, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p"), }, }, qu: `SELECT subject_hash AS ` + tagNode + `, object_hash AS o1, object_hash AS o2, label_hash AS "l 1" FROM quads WHERE predicate_hash = $1`, args: sVals("p"), }, { name: "quad actions and save", s: shape.Save{ Tags: []string{"sub"}, From: shape.QuadsAction{ Result: quad.Subject, Save: map[quad.Direction][]string{ quad.Object: {"o1", "o2"}, quad.Label: {"l 1"}, }, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p"), }, }, }, qu: `SELECT subject_hash AS sub, subject_hash AS ` + tagNode + `, object_hash AS o1, object_hash AS o2, label_hash AS "l 1" FROM quads WHERE predicate_hash = $1`, args: sVals("p"), }, { name: "quads with subquery", s: shape.Quads{ {Dir: quad.Subject, Values: shape.Fixed{sVal("s")}}, { Dir: quad.Predicate, Values: shape.QuadsAction{ Result: quad.Subject, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p"), }, }, }, }, qu: `SELECT t_1.subject_hash AS __subject, t_1.predicate_hash AS __predicate, t_1.object_hash AS __object, t_1.label_hash AS __label FROM quads AS t_1, (SELECT subject_hash AS ` + tagNode + ` FROM quads WHERE predicate_hash = $1) AS t_2 WHERE t_1.subject_hash = $2 AND t_1.predicate_hash = t_2.` + tagNode, args: sVals("p", "s"), }, { name: "quads with subquery (inner tags)", s: shape.Quads{ {Dir: quad.Subject, Values: shape.Fixed{sVal("s")}}, { Dir: quad.Predicate, Values: shape.Save{ Tags: []string{"pred"}, From: shape.QuadsAction{ Result: quad.Subject, Save: map[quad.Direction][]string{ quad.Object: {"ob"}, }, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p"), }, }, }, }, }, qu: `SELECT t_1.subject_hash AS __subject, t_1.predicate_hash AS __predicate, t_1.object_hash AS __object, t_1.label_hash AS __label, t_2.subject_hash AS pred, t_2.object_hash AS ob FROM quads AS t_1, quads AS t_2 WHERE t_1.subject_hash = $1 AND t_2.predicate_hash = $2 AND t_1.predicate_hash = t_2.subject_hash`, args: sVals("s", "p"), }, { name: "quads with subquery (limit)", s: shape.Quads{ {Dir: quad.Subject, Values: shape.Fixed{sVal("s")}}, { Dir: quad.Predicate, Values: shape.Page{ Limit: 10, From: shape.QuadsAction{ Result: quad.Subject, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p"), }, }, }, }, }, qu: `SELECT t_1.subject_hash AS __subject, t_1.predicate_hash AS __predicate, t_1.object_hash AS __object, t_1.label_hash AS __label FROM quads AS t_1, (SELECT subject_hash AS ` + tagNode + ` FROM quads WHERE predicate_hash = $1 LIMIT 10) AS t_2 WHERE t_1.subject_hash = $2 AND t_1.predicate_hash = t_2.` + tagNode, args: sVals("p", "s"), }, { skip: true, // TODO name: "quads with subquery (inner tags + limit)", s: shape.Quads{ {Dir: quad.Subject, Values: shape.Fixed{sVal("s")}}, { Dir: quad.Predicate, Values: shape.Save{ Tags: []string{"pred"}, From: shape.Page{ Limit: 10, From: shape.QuadsAction{ Result: quad.Subject, Save: map[quad.Direction][]string{ quad.Object: {"ob"}, }, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p"), }, }, }, }, }, }, qu: ``, args: []Value{}, }, { name: "nodes from quads", s: shape.NodesFrom{ Dir: quad.Object, Quads: shape.Quads{ {Dir: quad.Subject, Values: shape.Fixed{sVal("s")}}, { Dir: quad.Predicate, Values: shape.QuadsAction{ Result: quad.Subject, Save: map[quad.Direction][]string{ quad.Object: {"ob"}, }, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p"), }, }, }, }, }, qu: `SELECT t_1.object_hash AS ` + tagNode + `, t_2.object_hash AS ob FROM quads AS t_1, quads AS t_2 WHERE t_1.subject_hash = $1 AND t_2.predicate_hash = $2 AND t_1.predicate_hash = t_2.subject_hash`, args: sVals("s", "p"), }, { name: "intersect selects", s: shape.Intersect{ shape.Save{ Tags: []string{"sub"}, From: shape.QuadsAction{ Result: quad.Subject, Save: map[quad.Direction][]string{ quad.Object: {"o1"}, quad.Label: {"l 1"}, }, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p1"), }, }, }, shape.NodesFrom{ Dir: quad.Object, Quads: shape.Quads{ {Dir: quad.Subject, Values: shape.Fixed{sVal("s")}}, { Dir: quad.Predicate, Values: shape.QuadsAction{ Result: quad.Subject, Save: map[quad.Direction][]string{ quad.Object: {"ob"}, }, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("p2"), }, }, }, }, }, }, qu: `SELECT t_3.subject_hash AS sub, t_3.subject_hash AS __node, t_3.object_hash AS o1, t_3.label_hash AS "l 1", t_2.object_hash AS ob FROM quads AS t_3, quads AS t_1, quads AS t_2 WHERE t_3.predicate_hash = $1 AND t_1.subject_hash = $2 AND t_2.predicate_hash = $3 AND t_1.predicate_hash = t_2.subject_hash AND t_3.subject_hash = t_1.object_hash`, args: sVals("p1", "s", "p2"), }, { name: "deep shape", s: shape.NodesFrom{ Dir: quad.Object, Quads: shape.Quads{ shape.QuadFilter{Dir: quad.Predicate, Values: shape.Fixed{sVal("s")}}, shape.QuadFilter{ Dir: quad.Subject, Values: shape.NodesFrom{ Dir: quad.Subject, Quads: shape.Quads{ shape.QuadFilter{Dir: quad.Predicate, Values: shape.Fixed{sVal("s")}}, shape.QuadFilter{ Dir: quad.Object, Values: shape.NodesFrom{ Dir: quad.Subject, Quads: shape.Quads{ shape.QuadFilter{Dir: quad.Predicate, Values: shape.Fixed{sVal("a")}}, shape.QuadFilter{ Dir: quad.Object, Values: shape.QuadsAction{ Result: quad.Subject, Filter: map[quad.Direction]graph.Ref{ quad.Predicate: sVal("n"), quad.Object: sVal("k"), }, }, }, }, }, }, }, }, }, }, }, qu: `SELECT t_5.object_hash AS __node FROM quads AS t_5, (SELECT t_3.subject_hash AS __node FROM quads AS t_3, (SELECT t_1.subject_hash AS __node FROM quads AS t_1, (SELECT subject_hash AS __node FROM quads WHERE predicate_hash = $1 AND object_hash = $2) AS t_2 WHERE t_1.predicate_hash = $3 AND t_1.object_hash = t_2.__node) AS t_4 WHERE t_3.predicate_hash = $4 AND t_3.object_hash = t_4.__node) AS t_6 WHERE t_5.predicate_hash = $5 AND t_5.subject_hash = t_6.__node`, args: sVals("n", "k", "a", "s", "s"), }, } func TestSQLShapes(t *testing.T) { dialect := DefaultDialect dialect.Placeholder = func(i int) string { return fmt.Sprintf("$%d", i) } for _, c := range shapeCases { t.Run(c.name, func(t *testing.T) { opt := NewOptimizer() s, ok := c.s.Optimize(context.TODO(), opt) if c.skip { t.Skipf("%#v", s) } require.True(t, ok, "%#v", s) sq, ok := s.(Shape) require.True(t, ok, "%#v", s) b := NewBuilder(dialect) require.Equal(t, c.qu, sq.SQL(b), "%#v", sq) require.Equal(t, c.args, sq.Args()) }) } } ================================================ FILE: graph/sql/sqlite/sqlite.go ================================================ //go:build cgo package sqlite import ( "database/sql" "fmt" "regexp" "strings" "github.com/cayleygraph/quad" sqlite3 "github.com/mattn/go-sqlite3" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" graphlog "github.com/cayleygraph/cayley/graph/log" csql "github.com/cayleygraph/cayley/graph/sql" ) const Type = "sqlite" var QueryDialect = csql.QueryDialect{ RegexpOp: "REGEXP", FieldQuote: func(name string) string { return "`" + name + "`" }, Placeholder: func(n int) string { return "?" }, } func init() { regex := func(re, s string) (bool, error) { return regexp.MatchString(re, s) } sql.Register("sqlite3-regexp", &sqlite3.SQLiteDriver{ ConnectHook: func(conn *sqlite3.SQLiteConn) error { return conn.RegisterFunc("regexp", regex, true) }, }) csql.Register(Type, csql.Registration{ Driver: "sqlite3-regexp", HashType: fmt.Sprintf(`BINARY(%d)`, quad.HashSize), BytesType: `BLOB`, HorizonType: `INTEGER`, TimeType: `DATETIME`, QueryDialect: QueryDialect, NoOffsetWithoutLimit: true, NoForeignKeys: true, Error: func(err error) error { return err }, Estimated: nil, RunTx: runTxSqlite, }) } func runTxSqlite(tx *sql.Tx, nodes []graphlog.NodeUpdate, quads []graphlog.QuadUpdate, opts graph.IgnoreOpts) error { // update node ref counts and insert nodes var ( // prepared statements for each value type insertValue = make(map[csql.ValueType]*sql.Stmt) updateValue *sql.Stmt ) for _, n := range nodes { if n.RefInc >= 0 { nodeKey, values, err := csql.NodeValues(csql.NodeHash{ValueHash: n.Hash}, n.Val) if err != nil { return err } values = append([]interface{}{n.RefInc}, values...) values = append(values, n.RefInc) // one more time for UPDATE stmt, ok := insertValue[nodeKey] if !ok { var ph = make([]string, len(values)-1) // excluding last increment for i := range ph { ph[i] = "?" } stmt, err = tx.Prepare(`INSERT INTO nodes(refs, hash, ` + strings.Join(nodeKey.Columns(), ", ") + `) VALUES (` + strings.Join(ph, ", ") + `) ON CONFLICT(hash) DO UPDATE SET refs = refs + ?;`) if err != nil { return err } insertValue[nodeKey] = stmt } _, err = stmt.Exec(values...) err = convInsertError(err) if err != nil { clog.Errorf("couldn't exec INSERT statement: %v", err) return err } } else { panic("unexpected node update") } } for _, s := range insertValue { s.Close() } if s := updateValue; s != nil { s.Close() } insertValue = nil updateValue = nil // now we can deal with quads ignore := "" if opts.IgnoreDup { ignore = " OR IGNORE" } var ( insertQuad *sql.Stmt err error ) for _, d := range quads { dirs := make([]interface{}, 0, len(quad.Directions)) for _, h := range d.Quad.Dirs() { dirs = append(dirs, csql.NodeHash{ValueHash: h}.SQLValue()) } if !d.Del { if insertQuad == nil { insertQuad, err = tx.Prepare(`INSERT` + ignore + ` INTO quads(subject_hash, predicate_hash, object_hash, label_hash, ts) VALUES (?, ?, ?, ?, datetime());`) if err != nil { return err } insertValue = make(map[csql.ValueType]*sql.Stmt) } _, err := insertQuad.Exec(dirs...) err = convInsertError(err) if err != nil { if _, ok := err.(*graph.DeltaError); !ok { clog.Errorf("couldn't exec INSERT statement: %v", err) } return err } } else { panic("unexpected quad delete") } } return nil } func convInsertError(err error) error { if err == nil { return nil } if e, ok := err.(sqlite3.Error); ok { if e.Code == sqlite3.ErrConstraint { return &graph.DeltaError{Err: graph.ErrQuadExists} } } return err } ================================================ FILE: graph/sql/sqlite/sqlite_test.go ================================================ //go:build cgo package sqlite import ( "fmt" "os" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/sql/sqltest" ) func makeSqlite(t testing.TB) (string, graph.Options) { tmpFile, err := os.CreateTemp("", fmt.Sprintf("cayley_test_%s*", Type)) if err != nil { t.Fatalf("Could not create working directory: %v", err) } t.Cleanup(func() { os.RemoveAll(tmpFile.Name()) }) return fmt.Sprintf("file:%s?_loc=UTC", tmpFile.Name()), nil } var conf = &sqltest.Config{ TimeRound: true, TimeInMcs: false, } func TestSqlite(t *testing.T) { sqltest.TestAll(t, Type, makeSqlite, conf) } func BenchmarkSqlite(t *testing.B) { sqltest.BenchmarkAll(t, Type, makeSqlite, conf) } ================================================ FILE: graph/sql/sqltest/sqltest.go ================================================ package sqltest import ( "testing" "unicode/utf8" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest" "github.com/cayleygraph/cayley/graph/graphtest/testutil" "github.com/cayleygraph/cayley/graph/sql" ) type Config struct { TimeRound bool TimeInMcs bool } func (c Config) quadStore() *graphtest.Config { return &graphtest.Config{ NoPrimitives: true, TimeInMcs: c.TimeInMcs, TimeRound: c.TimeRound, OptimizesComparison: true, } } func TestAll(t *testing.T, typ string, fnc DatabaseFunc, c *Config) { if c == nil { c = &Config{TimeInMcs: true} } create := makeDatabaseFunc(typ, fnc) t.Run("qs", func(t *testing.T) { t.Parallel() graphtest.TestAll(t, create, c.quadStore()) }) t.Run("zero rune", func(t *testing.T) { t.Parallel() testZeroRune(t, create) }) } func BenchmarkAll(t *testing.B, typ string, fnc DatabaseFunc, c *Config) { if c == nil { c = &Config{} } create := makeDatabaseFunc(typ, fnc) t.Run("qs", func(t *testing.B) { graphtest.BenchmarkAll(t, create, c.quadStore()) }) } type DatabaseFunc func(t testing.TB) (string, graph.Options) func makeDatabaseFunc(typ string, create DatabaseFunc) testutil.DatabaseFunc { return func(t testing.TB) (graph.QuadStore, graph.Options) { addr, opts := create(t) if err := sql.Init(typ, addr, opts); err != nil { t.Fatal(err) } qs, err := sql.New(typ, addr, opts) if err != nil { t.Fatal(err) } t.Cleanup(func() { qs.Close() }) return qs, nil } } func testZeroRune(t testing.TB, create testutil.DatabaseFunc) { qs, opts := create(t) w := testutil.MakeWriter(t, qs, opts) obj := quad.String("AB\u0000CD") if !utf8.ValidString(string(obj)) { t.Fatal("invalid utf8") } err := w.AddQuad(quad.Quad{ Subject: quad.IRI("bob"), Predicate: quad.IRI("pred"), Object: obj, }) require.NoError(t, err) qsv, err := qs.ValueOf(quad.Raw(obj.String())) require.NoError(t, err) qsn, err := qs.NameOf(qsv) require.NoError(t, err) require.Equal(t, obj, qsn) } ================================================ FILE: graph/transaction.go ================================================ // Copyright 2015 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package graph import "github.com/cayleygraph/quad" // Transaction stores a bunch of Deltas to apply together in an atomic step on the database. type Transaction struct { // Deltas stores the deltas in the right order Deltas []Delta // deltas stores the deltas in a map to avoid duplications deltas map[Delta]struct{} } // NewTransaction initialize a new transaction. func NewTransaction() *Transaction { return NewTransactionN(10) } // NewTransactionN initialize a new transaction with a predefined capacity. func NewTransactionN(n int) *Transaction { return &Transaction{Deltas: make([]Delta, 0, n), deltas: make(map[Delta]struct{}, n)} } // AddQuad adds a new quad to the transaction if it is not already present in it. // If there is a 'remove' delta for that quad, it will remove that delta from // the transaction instead of actually adding the quad. func (t *Transaction) AddQuad(q quad.Quad) { ad, rd := createDeltas(q) if _, adExists := t.deltas[ad]; !adExists { if _, rdExists := t.deltas[rd]; rdExists { t.deleteDelta(rd) } else { t.addDelta(ad) } } } // RemoveQuad adds a quad to remove to the transaction. // The quad will be removed from the database if it is not present in the // transaction, otherwise it simply remove it from the transaction. func (t *Transaction) RemoveQuad(q quad.Quad) { ad, rd := createDeltas(q) if _, adExists := t.deltas[ad]; adExists { t.deleteDelta(ad) } else { if _, rdExists := t.deltas[rd]; !rdExists { t.addDelta(rd) } } } func createDeltas(q quad.Quad) (ad, rd Delta) { ad = Delta{ Quad: q, Action: Add, } rd = Delta{ Quad: q, Action: Delete, } return } func (t *Transaction) addDelta(d Delta) { t.Deltas = append(t.Deltas, d) t.deltas[d] = struct{}{} } func (t *Transaction) deleteDelta(d Delta) { delete(t.deltas, d) for i, id := range t.Deltas { if id == d { t.Deltas = append(t.Deltas[:i], t.Deltas[i+1:]...) break } } } ================================================ FILE: graph/transaction_test.go ================================================ package graph import ( "testing" "github.com/cayleygraph/quad" ) func TestTransaction(t *testing.T) { var tx *Transaction // simples adds / removes tx = NewTransaction() tx.AddQuad(quad.Make("E", "follows", "F", nil)) tx.AddQuad(quad.Make("F", "follows", "G", nil)) tx.RemoveQuad(quad.Make("A", "follows", "Z", nil)) if len(tx.Deltas) != 3 { t.Errorf("Expected 3 Deltas, have %d delta(s)", len(tx.Deltas)) } // add, remove -> nothing tx = NewTransaction() tx.AddQuad(quad.Make("E", "follows", "G", nil)) tx.RemoveQuad(quad.Make("E", "follows", "G", nil)) if len(tx.Deltas) != 0 { t.Errorf("Expected [add, remove]->[], have %d Deltas", len(tx.Deltas)) } // remove, add -> nothing tx = NewTransaction() tx.RemoveQuad(quad.Make("E", "follows", "G", nil)) tx.AddQuad(quad.Make("E", "follows", "G", nil)) if len(tx.Deltas) != 0 { t.Errorf("Expected [add, remove]->[], have %d delta(s)", len(tx.Deltas)) } // add x2 -> add x1 tx = NewTransaction() tx.AddQuad(quad.Make("E", "follows", "G", nil)) tx.AddQuad(quad.Make("E", "follows", "G", nil)) if len(tx.Deltas) != 1 { t.Errorf("Expected [add, add]->[add], have %d delta(s)", len(tx.Deltas)) } // remove x2 -> remove x1 tx = NewTransaction() tx.RemoveQuad(quad.Make("E", "follows", "G", nil)) tx.RemoveQuad(quad.Make("E", "follows", "G", nil)) if len(tx.Deltas) != 1 { t.Errorf("Expected [remove, remove]->[remove], have %d delta(s)", len(tx.Deltas)) } // add, remove x2 -> remove x1 tx = NewTransaction() tx.AddQuad(quad.Make("E", "follows", "G", nil)) tx.RemoveQuad(quad.Make("E", "follows", "G", nil)) tx.RemoveQuad(quad.Make("E", "follows", "G", nil)) if len(tx.Deltas) != 1 { t.Errorf("Expected [add, remove, remove]->[remove], have %d delta(s)", len(tx.Deltas)) } } ================================================ FILE: imports.go ================================================ package cayley import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" _ "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/query/path" _ "github.com/cayleygraph/cayley/writer" "github.com/cayleygraph/quad" ) var ( StartMorphism = path.StartMorphism StartPath = path.StartPath NewTransaction = graph.NewTransaction ) type Iterator = iterator.Shape type QuadStore = graph.QuadStore type QuadWriter = graph.QuadWriter type Path = path.Path type Handle struct { graph.QuadStore graph.QuadWriter } func (h *Handle) Close() error { err := h.QuadWriter.Close() h.QuadStore.Close() return err } func Triple(subject, predicate, object interface{}) quad.Quad { return Quad(subject, predicate, object, nil) } func Quad(subject, predicate, object, label interface{}) quad.Quad { return quad.Make(subject, predicate, object, label) } func NewGraph(name, dbpath string, opts graph.Options) (*Handle, error) { qs, err := graph.NewQuadStore(name, dbpath, opts) if err != nil { return nil, err } qw, err := graph.NewQuadWriter("single", qs, nil) if err != nil { return nil, err } return &Handle{qs, qw}, nil } func NewMemoryGraph() (*Handle, error) { return NewGraph("memstore", "", nil) } ================================================ FILE: inference/inference.go ================================================ // Package inference implements an in-memory store for inference. // // RDFS Rules: // // 1. (x p y) -> (p rdf:type rdf:Property) // 2. (p rdfs:domain c), (x p y) -> (x rdf:type c) // 3. (p rdfs:range c), (x p y) -> (y rdf:type c) // 4a. (x p y) -> (x rdf:type rdfs:Resource) // 4b. (x p y) -> (y rdf:type rdfs:Resource) // 5. (p rdfs:subPropertyOf q), (q rdfs:subPropertyOf r) -> (p rdfs:subPropertyOf r) // 6. (p rdf:type Property) -> (p rdfs:subPropertyOf p) // 7. (p rdf:subPropertyOf q), (x p y) -> (x q y) // 8. (c rdf:type rdfs:Class) -> (c rdfs:subClassOf rdfs:Resource) // 9. (c rdfs:subClassOf d), (x rdf:type c) -> (x rdf:type d) // 10. (c rdf:type rdfs:Class) -> (c rdfs:subClassOf c) // 11. (c rdfs:subClassOf d), (d rdfs:subClassOf e) -> (c rdfs:subClassOf e) // 12. (p rdf:type rdfs:ContainerMembershipProperty) -> (p rdfs:subPropertyOf rdfs:member) // 13. (x rdf:type rdfs:Datatype) -> (x rdfs:subClassOf rdfs:Literal) // // Exported from: https://www.researchgate.net/figure/RDF-RDFS-entailment-rules_tbl1_268419911 // // Implemented here: 1 2 3 5 6 8 10 11 package inference import ( "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc/rdf" "github.com/cayleygraph/quad/voc/rdfs" ) // classSet is a set of RDF Classes type classSet map[*Class]struct{} // propertySet is a set of RDF Properties type propertySet map[*Property]struct{} // Class represents a RDF Class with the links to classes and other properties type Class struct { store *Store name quad.Value explicit bool references int super classSet sub classSet ownProp propertySet inProp propertySet } func (s *Store) newClass(name quad.Value, explicit bool) *Class { c := &Class{ store: s, name: name, explicit: explicit, super: make(classSet), sub: make(classSet), ownProp: make(propertySet), inProp: make(propertySet), } s.classes[name] = c return c } // Name returns the class's name func (c *Class) Name() quad.Value { return c.name } // IsSubClassOf recursively checks whether class is a superClass func (c *Class) IsSubClassOf(super *Class) bool { if c == super { return true } if super.name == quad.IRI(rdfs.Resource) { return true } if _, ok := c.super[super]; ok { return true } for s := range c.super { if s.IsSubClassOf(super) { return true } } return false } func (c *Class) isReferenced() bool { return c.explicit || len(c.super) > 0 || len(c.sub) > 0 || len(c.ownProp) > 0 || len(c.inProp) > 0 || c.references > 0 } func (c *Class) deleteIfUnreferenced() { if c != nil && !c.isReferenced() { c.store.deleteClass(c.name) } } // Property represents a RDF Property with the links to classes and other properties type Property struct { name quad.Value explicit bool references int domain *Class prange *Class super propertySet sub propertySet store *Store } func newProperty(name quad.Value, explicit bool, store *Store) *Property { return &Property{ name: name, explicit: explicit, super: make(propertySet), sub: make(propertySet), store: store, } } // Name returns the property's name func (p *Property) Name() quad.Value { return p.name } // Domain returns the domain of the property func (p *Property) Domain() *Class { return p.domain } // Range returns the range of the property func (p *Property) Range() *Class { return p.prange } // IsSubPropertyOf recursively checks whether property is a superProperty func (p *Property) IsSubPropertyOf(super *Property) bool { if p == super { return true } if _, ok := p.super[super]; ok { return true } for s := range p.super { if s.IsSubPropertyOf(super) { return true } } return false } func (p *Property) isReferenced() bool { return p.explicit || p.references > 0 || len(p.super) > 0 || len(p.sub) > 0 || p.domain != nil || p.prange != nil } func (p *Property) deleteIfUnreferenced() { if p != nil && !p.isReferenced() { p.store.deleteProperty(p.name) } } // Store is a struct holding the inference data type Store struct { classes map[quad.Value]*Class properties map[quad.Value]*Property } // NewStore creates a new Store func NewStore() Store { s := Store{ classes: make(map[quad.Value]*Class), properties: make(map[quad.Value]*Property), } s.ensureClass(quad.IRI(rdfs.Resource)) return s } // GetClass returns a class struct for class name, if it doesn't exist in the store then it returns nil func (s *Store) GetClass(name quad.Value) *Class { return s.classes[name] } // GetProperty returns a class struct for property name, if it doesn't exist in the store then it returns nil func (s *Store) GetProperty(name quad.Value) *Property { return s.properties[name] } func (s *Store) ensureClass(name quad.Value) { if c, ok := s.classes[name]; ok { c.explicit = true } else { _ = s.newClass(name, true) } } func (s *Store) getOrCreateImplicitClass(name quad.Value) *Class { c, ok := s.classes[name] if !ok { c = s.newClass(name, false) } return c } func (s *Store) createProperty(name quad.Value) { if property, ok := s.properties[name]; ok { property.explicit = true return } s.properties[name] = newProperty(name, true, s) } func (s *Store) getOrCreateImplicitProperty(name quad.Value) *Property { if p, ok := s.properties[name]; ok { return p } p := newProperty(name, false, s) s.properties[name] = p return p } func (s *Store) addClassRelationship(child quad.Value, parent quad.Value) { p := s.getOrCreateImplicitClass(parent) c := s.getOrCreateImplicitClass(child) if _, ok := p.sub[c]; !ok { p.sub[c] = struct{}{} c.super[p] = struct{}{} } } func (s *Store) addPropertyRelationship(child quad.Value, parent quad.Value) { p := s.getOrCreateImplicitProperty(parent) c := s.getOrCreateImplicitProperty(child) if _, ok := p.sub[c]; !ok { p.sub[c] = struct{}{} c.super[p] = struct{}{} } } func (s *Store) setPropertyDomain(property quad.Value, domain quad.Value) { p := s.getOrCreateImplicitProperty(property) c := s.getOrCreateImplicitClass(domain) // FIXME(iddan): Currently doesn't support multiple domains as they are very rare p.domain = c c.ownProp[p] = struct{}{} } func (s *Store) setPropertyRange(property quad.Value, prange quad.Value) { p := s.getOrCreateImplicitProperty(property) c := s.getOrCreateImplicitClass(prange) p.prange = c // FIXME(iddan): Currently doesn't support multiple ranges as they are very rare c.inProp[p] = struct{}{} } func (s *Store) addClassInstance(name quad.Value) { c := s.GetClass(name) if c == nil { c = s.getOrCreateImplicitClass(name) } c.references++ } func (s *Store) addPropertyInstance(name quad.Value) *Property { p := s.GetProperty(name) if p == nil { p = s.getOrCreateImplicitProperty(name) } p.references++ return p } // ProcessQuads is used to update the store with multiple quads func (s *Store) ProcessQuads(quads ...quad.Quad) { for _, q := range quads { s.processQuad(q) } } // processQuad is used to update the store with a new quad func (s *Store) processQuad(q quad.Quad) { pred, ok := q.Predicate.(quad.IRI) if !ok { return } sub, obj := q.Subject, q.Object switch pred { case rdf.Type: switch obj := obj.(type) { case quad.BNode: s.addClassInstance(obj) case quad.IRI: switch obj { case rdfs.Class: s.ensureClass(sub) case rdf.Property: s.createProperty(sub) default: s.addClassInstance(obj) } } case rdfs.SubPropertyOf: s.addPropertyRelationship(sub, obj) case rdfs.SubClassOf: s.addClassRelationship(sub, obj) case rdfs.Domain: s.setPropertyDomain(sub, obj) case rdfs.Range: s.setPropertyRange(sub, obj) default: p := s.addPropertyInstance(pred) domain := p.Domain() if domain != nil { domain.references++ } prange := p.Range() if prange != nil { prange.references++ } } } func (s *Store) deleteClass(name quad.Value) { c, ok := s.classes[name] if !ok { return } for sub := range c.sub { delete(sub.super, c) } for super := range c.super { delete(super.sub, c) } delete(s.classes, name) } func (s *Store) deleteProperty(name quad.Value) { p, ok := s.properties[name] if !ok { return } for super := range p.super { delete(super.sub, p) } for sub := range p.sub { delete(sub.super, p) } delete(s.properties, name) } func (s *Store) deleteClassRel(child quad.Value, parent quad.Value) { p := s.GetClass(parent) c := s.GetClass(child) if _, ok := p.sub[c]; ok { delete(p.sub, c) delete(c.super, p) p.deleteIfUnreferenced() c.deleteIfUnreferenced() } } func (s *Store) deletePropertyRel(child quad.Value, parent quad.Value) { p := s.GetProperty(parent) c := s.GetProperty(child) if _, ok := p.sub[c]; ok { delete(p.sub, c) delete(c.super, p) p.deleteIfUnreferenced() c.deleteIfUnreferenced() } } func (s *Store) unsetPropertyDomain(property quad.Value, domain quad.Value) { p := s.GetProperty(property) c := s.GetClass(domain) // FIXME(iddan): Currently doesn't support multiple domains as they are very rare p.domain = nil delete(c.ownProp, p) p.deleteIfUnreferenced() c.deleteIfUnreferenced() } func (s *Store) unsetPropertyRange(property quad.Value, prange quad.Value) { p := s.GetProperty(property) c := s.GetClass(prange) p.prange = nil // FIXME(iddan): Currently doesn't support multiple ranges as they are very rare delete(c.inProp, p) p.deleteIfUnreferenced() c.deleteIfUnreferenced() } func (s *Store) deleteClassInstance(name quad.Value) { c := s.GetClass(name) if c == nil { return } c.references-- c.deleteIfUnreferenced() } func (s *Store) deletePropertyInstance(name quad.Value) *Property { p := s.GetProperty(name) if p == nil { return nil } p.references-- p.deleteIfUnreferenced() return p } // UnprocessQuads is used to delete multiple quads from the store func (s *Store) UnprocessQuads(quads ...quad.Quad) { for _, q := range quads { s.unprocessQuad(q) } } // unprocessQuad is used to delete a quad from the store func (s *Store) unprocessQuad(q quad.Quad) { pred, ok := q.Predicate.(quad.IRI) if !ok { return } sub, obj := q.Subject, q.Object switch pred { case rdf.Type: obj, ok := obj.(quad.IRI) if !ok { return } switch obj { case rdfs.Class: s.deleteClass(sub) case rdf.Property: s.deleteProperty(sub) default: s.deleteClassInstance(obj) } case rdfs.SubPropertyOf: s.deletePropertyRel(sub, obj) case rdfs.SubClassOf: s.deleteClassRel(sub, obj) case rdfs.Domain: s.unsetPropertyDomain(sub, obj) case rdfs.Range: s.unsetPropertyRange(sub, obj) default: p := s.deletePropertyInstance(pred) if p != nil { if domain := p.Domain(); domain != nil { s.deleteClassInstance(domain.Name()) } if prange := p.Range(); prange != nil { s.deleteClassInstance(prange.Name()) } } } } ================================================ FILE: inference/inference_test.go ================================================ package inference import ( "testing" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc/rdf" "github.com/cayleygraph/quad/voc/rdfs" "github.com/stretchr/testify/require" ) func triple(subject quad.Value, predicate quad.IRI, object quad.Value) quad.Quad { return quad.Quad{Subject: subject, Predicate: predicate, Object: object} } var ( domain = quad.IRI(rdfs.Domain) prange = quad.IRI(rdfs.Range) ptype = quad.IRI(rdf.Type) class = quad.IRI(rdfs.Class) literal = quad.IRI(rdfs.Literal) property = quad.IRI(rdf.Property) subClassOf = quad.IRI(rdfs.SubClassOf) subPropertyOf = quad.IRI(rdfs.SubPropertyOf) alice = quad.IRI("alice") aliceName = quad.String("Alice") bob = quad.IRI("bob") engineer = quad.IRI("Engineer") information = quad.IRI("information") likes = quad.IRI("likes") name = quad.IRI("name") person = quad.IRI("Person") personal = quad.IRI("personal") softwareEngineer = quad.IRI("SoftwareEngineer") ) var ( aliceIsPerson = triple(alice, ptype, person) aliceLikesBob = triple(alice, likes, bob) likesRangePerson = triple(likes, prange, person) engineerClass = triple(engineer, ptype, class) engineerSubClass = triple(engineer, subClassOf, person) nameDomainPerson = triple(name, domain, person) nameProperty = triple(name, ptype, property) nameSubPropertyOfPersonal = triple(name, subPropertyOf, personal) personalProperty = triple(personal, ptype, property) personalSubPropertyOfInformation = triple(personal, subPropertyOf, information) personClass = triple(person, ptype, class) softwareEngineerClass = triple(softwareEngineer, ptype, class) aliceNameAlice = triple(alice, name, aliceName) ) var ( engineerAndSoftwareEngineerSubClasses = []quad.Quad{ engineerSubClass, triple(softwareEngineer, subClassOf, engineer), } engineerAndPersonClasses = []quad.Quad{ engineerClass, personClass, } ) func TestClassName(t *testing.T) { iri := alice c := Class{name: iri} require.Equal(t, c.Name(), iri, "Name was not set correctly for the class") } func TestPropertyName(t *testing.T) { iri := likes p := Property{name: iri} require.Equal(t, p.Name(), iri, "Name was not set correctly for the property") } func TestReferencedType(t *testing.T) { store := NewStore() q := aliceIsPerson store.ProcessQuads(q) createdClass := store.GetClass(person) require.NotNil(t, createdClass, "Class was not created") } func TestReferencedBNodeType(t *testing.T) { store := NewStore() name := quad.BNode("123") q := triple(alice, ptype, name) store.ProcessQuads(q) createdClass := store.GetClass(name) require.NotNil(t, createdClass, "Class was not created") } func TestReferencedProperty(t *testing.T) { store := NewStore() q := aliceLikesBob store.ProcessQuads(q) createdProperty := store.GetProperty(likes) require.NotNil(t, createdProperty, "Property was not created") } func TestNewClass(t *testing.T) { store := NewStore() q := personClass store.ProcessQuads(q) createdClass := store.GetClass(person) require.NotNil(t, createdClass, "Class was not created") } func TestNewBNodeClass(t *testing.T) { store := NewStore() name := quad.BNode("123") q := triple(name, ptype, class) store.ProcessQuads(q) createdClass := store.GetClass(name) require.NotNil(t, createdClass, "Class was not created") } func TestInvalidNewClass(t *testing.T) { store := NewStore() name := quad.String("Foo") q := triple(alice, ptype, name) store.ProcessQuads(q) createdClass := store.GetClass(name) require.Nil(t, createdClass, "Invalid class was created") } func TestNewProperty(t *testing.T) { store := NewStore() q := nameProperty store.ProcessQuads(q) createdProperty := store.GetProperty(name) require.NotNil(t, createdProperty, "Property was not created") } func TestInvalidNewProperty(t *testing.T) { store := NewStore() name := quad.String("Foo") q := quad.Quad{Subject: alice, Predicate: name, Object: bob} store.ProcessQuads(q) createdProperty := store.GetProperty(name) require.Nil(t, createdProperty, "Invalid property was created") } func TestSubClass(t *testing.T) { store := NewStore() q := engineerSubClass store.ProcessQuads(q) createdClass := store.GetClass(engineer) createdSuperClass := store.GetClass(person) require.NotNil(t, createdClass, "Class was not created") require.NotNil(t, createdSuperClass, "Super class was not created") if _, ok := createdClass.super[createdSuperClass]; !ok { t.Error("Super class was not registered for class") } if _, ok := createdSuperClass.sub[createdClass]; !ok { t.Error("Class was not registered for super class") } } func TestSubProperty(t *testing.T) { store := NewStore() q := nameSubPropertyOfPersonal store.ProcessQuads(q) createdProperty := store.GetProperty(name) createdSuperProperty := store.GetProperty(personal) require.NotNil(t, createdProperty, "Property was not created") require.NotNil(t, createdSuperProperty, "Super property was not created") if _, ok := createdProperty.super[createdSuperProperty]; !ok { t.Error("Super property was not registered for property") } if _, ok := createdSuperProperty.sub[createdProperty]; !ok { t.Error("Property was not registered for super property") } } func TestPropertyDomain(t *testing.T) { store := NewStore() q := nameDomainPerson store.ProcessQuads(q) createdProperty := store.GetProperty(name) createdClass := store.GetClass(person) require.NotNil(t, createdProperty, "Property was not created") require.NotNil(t, createdClass, "Domain class was not created") if createdProperty.Domain() != createdClass { t.Error("Domain class was not registered for property") } if _, ok := createdClass.ownProp[createdProperty]; !ok { t.Error("Property was not registered for class") } } func TestPropertyRange(t *testing.T) { store := NewStore() q := likesRangePerson store.ProcessQuads(q) createdProperty := store.GetProperty(likes) createdClass := store.GetClass(person) require.NotNil(t, createdProperty, "Property was not created") require.NotNil(t, createdClass, "Range class was not created") if createdProperty.Range() != createdClass { t.Error("Range class was not registered for property") } if _, ok := createdClass.inProp[createdProperty]; !ok { t.Error("Property was not registered for class") } } func TestIsSubClassOf(t *testing.T) { store := NewStore() q := engineerSubClass store.ProcessQuads(q) if !store.GetClass(engineer).IsSubClassOf(store.GetClass(person)) { t.Error("Class was not registered as subclass of super class") } } func TestIsSubClassOfRecursive(t *testing.T) { store := NewStore() quads := engineerAndSoftwareEngineerSubClasses store.ProcessQuads(quads...) if !store.GetClass(softwareEngineer).IsSubClassOf(store.GetClass(person)) { t.Error("Class was not registered as subclass of super class") } } func TestIsSubClassOfItself(t *testing.T) { store := NewStore() q := personClass store.ProcessQuads(q) if !store.GetClass(person).IsSubClassOf(store.GetClass(person)) { t.Error("IsSubClassOf itself doesn't work") } } func TestIsSubClassOfResource(t *testing.T) { store := NewStore() q := personClass store.ProcessQuads(q) if !store.GetClass(person).IsSubClassOf(store.GetClass(quad.IRI(rdfs.Resource))) { t.Error("ItSubClassOf rdfs:Resource doesn't work") } } func TestIsSubPropertyOf(t *testing.T) { store := NewStore() q := nameSubPropertyOfPersonal store.ProcessQuads(q) if !store.GetProperty(name).IsSubPropertyOf(store.GetProperty(personal)) { t.Error("Property was not registered as subproperty of super property") } } func TestIsSubPropertyOfRecursive(t *testing.T) { store := NewStore() quads := []quad.Quad{ nameSubPropertyOfPersonal, personalSubPropertyOfInformation, } store.ProcessQuads(quads...) if !store.GetProperty(name).IsSubPropertyOf(store.GetProperty(information)) { t.Error("Property was not registered as subproperty of super property") } } func TestIsSubPropertyOfItself(t *testing.T) { store := NewStore() q := nameProperty store.ProcessQuads(q) if !store.GetProperty(name).IsSubPropertyOf(store.GetProperty(name)) { t.Error("IsSubPropertyOf itself doesn't work") } } func TestUnprocessInvalidQuad(t *testing.T) { store := NewStore() store.UnprocessQuads(quad.Quad{Subject: alice, Predicate: quad.String("Foo"), Object: person}) } func TestUnprocessInvalidTypeQuad(t *testing.T) { store := NewStore() store.UnprocessQuads(quad.Quad{Subject: alice, Predicate: ptype, Object: quad.String("Foo")}) } func TestDeleteReferencedType(t *testing.T) { store := NewStore() q := aliceIsPerson store.ProcessQuads(q) store.UnprocessQuads(q) createdClass := store.GetClass(person) require.Nil(t, createdClass, "Class was not deleted") } func TestDeleteClassWithSubClass(t *testing.T) { store := NewStore() store.ProcessQuads( engineerClass, engineerSubClass, ) q := personClass store.ProcessQuads(q) store.UnprocessQuads(q) subClass := store.GetClass(engineer) if len(subClass.super) != 0 { t.Error("Class was not unreferenced") } } func TestDeleteClassWithSuperClass(t *testing.T) { store := NewStore() store.ProcessQuads( personClass, engineerSubClass, ) q := engineerClass store.ProcessQuads(q) store.UnprocessQuads(q) superClass := store.GetClass(person) if len(superClass.sub) != 0 { t.Error("Class was not unreferenced") } } func TestDeleteNewClass(t *testing.T) { store := NewStore() q := personClass store.ProcessQuads(q) store.UnprocessQuads(q) createdClass := store.GetClass(person) require.Nil(t, createdClass, "Class was not deleted") } func TestDeleteNewProperty(t *testing.T) { store := NewStore() q := nameProperty store.ProcessQuads(q) store.UnprocessQuads(q) createdProperty := store.GetProperty(name) require.Nil(t, createdProperty, "Property was not deleted") } func TestDeletePropertyWithSubProperty(t *testing.T) { store := NewStore() store.ProcessQuads( nameProperty, nameSubPropertyOfPersonal, ) q := personalProperty store.ProcessQuads(q) store.UnprocessQuads(q) subProperty := store.GetProperty(name) if len(subProperty.super) != 0 { t.Error("Property was not unreferenced") } } func TestDeletePropertyWithSuperProperty(t *testing.T) { store := NewStore() store.ProcessQuads( personalProperty, nameSubPropertyOfPersonal, ) q := nameProperty store.ProcessQuads(q) store.UnprocessQuads(q) superProperty := store.GetProperty(personal) if len(superProperty.sub) != 0 { t.Error("Property was not unreferenced") } } func TestDeleteSubClass(t *testing.T) { store := NewStore() store.ProcessQuads(engineerAndPersonClasses...) q := engineerSubClass store.ProcessQuads(q) store.UnprocessQuads(q) createdClass := store.GetClass(engineer) createdSuperClass := store.GetClass(person) // TODO(iddan): what about garbage collection? if _, ok := createdClass.super[createdSuperClass]; ok { t.Error("Super class was not unregistered for class") } if _, ok := createdSuperClass.sub[createdClass]; ok { t.Error("Class was not unregistered for super class") } } func TestDeleteSubProperty(t *testing.T) { store := NewStore() store.ProcessQuads( nameProperty, personalProperty, ) q := nameSubPropertyOfPersonal store.ProcessQuads(q) store.UnprocessQuads(q) createdProperty := store.GetProperty(name) createdSuperProperty := store.GetProperty(personal) // TODO(iddan): what about garbage collection? if _, ok := createdProperty.super[createdSuperProperty]; ok { t.Error("Super property was not unregistered for property") } if _, ok := createdSuperProperty.sub[createdProperty]; ok { t.Error("Property was not unregistered for super property") } } func TestDeletePropertyDomain(t *testing.T) { store := NewStore() store.ProcessQuads( nameProperty, personClass, ) q := nameDomainPerson store.ProcessQuads(q) store.UnprocessQuads(q) createdProperty := store.GetProperty(name) createdClass := store.GetClass(person) // TODO(iddan): what about garbage collection? if createdProperty.Domain() == createdClass { t.Error("Domain class was not unregistered for property") } if _, ok := createdClass.ownProp[createdProperty]; ok { t.Error("Property was not unregistered for class") } } func TestDeletePropertyRange(t *testing.T) { store := NewStore() store.ProcessQuads( nameProperty, quad.Quad{Subject: literal, Predicate: ptype, Object: class}, ) q := quad.Quad{Subject: name, Predicate: prange, Object: literal} store.ProcessQuads(q) store.UnprocessQuads(q) createdProperty := store.GetProperty(name) createdClass := store.GetClass(literal) // TODO(iddan): what about garbage collection? if createdProperty.Range() == createdClass { t.Error("Range class was not unregistered for property") } if _, ok := createdClass.inProp[createdProperty]; ok { t.Error("Property was not unregistered for class") } } func TestDeleteIsSubClassOf(t *testing.T) { store := NewStore() store.ProcessQuads(engineerAndPersonClasses...) q := engineerSubClass store.ProcessQuads(q) store.UnprocessQuads(q) if store.GetClass(engineer).IsSubClassOf(store.GetClass(person)) { t.Error("Class was not unregistered as subclass of super class") } } func TestDeleteIsSubClassOfRecursive(t *testing.T) { store := NewStore() store.ProcessQuads( engineerClass, personClass, softwareEngineerClass, ) quads := engineerAndSoftwareEngineerSubClasses store.ProcessQuads(quads...) store.UnprocessQuads(quads...) if store.GetClass(softwareEngineer).IsSubClassOf(store.GetClass(person)) { t.Error("Class was not unregistered as subclass of super class") } } func TestDeleteIsSubPropertyOf(t *testing.T) { store := NewStore() store.ProcessQuads( nameProperty, personalProperty, ) q := nameSubPropertyOfPersonal store.ProcessQuads(q) store.UnprocessQuads(q) if store.GetProperty(name).IsSubPropertyOf(store.GetProperty(personal)) { t.Error("Property was not unregistered as subproperty of super property") } } func TestDeleteIsSubPropertyOfRecursive(t *testing.T) { store := NewStore() store.ProcessQuads( nameProperty, personalProperty, quad.Quad{Subject: information, Predicate: ptype, Object: property}, ) quads := []quad.Quad{ nameSubPropertyOfPersonal, personalSubPropertyOfInformation, } store.ProcessQuads(quads...) store.UnprocessQuads(quads...) if store.GetProperty(name).IsSubPropertyOf(store.GetProperty(information)) { t.Error("Property was not unregistered as subproperty of super property") } } func TestClassIsReference(t *testing.T) { store := NewStore() q := aliceIsPerson store.ProcessQuads(q) class := store.GetClass(person) if !class.isReferenced() { t.Error("Class should be referenced") } } func TestPropertyIsReference(t *testing.T) { store := NewStore() q := aliceLikesBob store.ProcessQuads(q) property := store.GetProperty(likes) if !property.isReferenced() { t.Error("Property should be referenced") } } func TestClassUnreference(t *testing.T) { store := NewStore() q := aliceIsPerson store.ProcessQuads(q) store.UnprocessQuads(q) require.Nil(t, store.GetClass(person), "class was not garbage collected") } func TestPropertyUnreference(t *testing.T) { store := NewStore() q := aliceLikesBob store.ProcessQuads(q) store.UnprocessQuads(q) require.Nil(t, store.GetProperty(likes), "property was not garbage collected") } func TestDomainClassInstance(t *testing.T) { store := NewStore() quads := []quad.Quad{ nameDomainPerson, aliceNameAlice, } store.ProcessQuads(quads...) class := store.GetClass(person) require.NotNil(t, class) require.True(t, class.isReferenced()) require.Equal(t, 1, class.references) store.UnprocessQuads(aliceNameAlice) require.True(t, class.isReferenced()) require.Equal(t, 0, class.references) store.UnprocessQuads(nameDomainPerson) require.False(t, class.isReferenced()) require.Equal(t, 0, class.references) } func TestRangeClassInstance(t *testing.T) { store := NewStore() quads := []quad.Quad{ likesRangePerson, aliceLikesBob, } store.ProcessQuads(quads...) class := store.GetClass(person) require.NotNil(t, class) require.Equal(t, 1, class.references) require.True(t, class.isReferenced()) store.UnprocessQuads(aliceLikesBob) require.Equal(t, 0, class.references) require.True(t, class.isReferenced()) store.UnprocessQuads(likesRangePerson) require.Equal(t, 0, class.references) require.False(t, class.isReferenced()) } func TestDeleteNonExistingClass(t *testing.T) { store := NewStore() store.UnprocessQuads(personClass) } func TestDeleteNonExistingProperty(t *testing.T) { store := NewStore() store.UnprocessQuads(personalProperty) } func TestDeleteNonExistingClassInstance(t *testing.T) { store := NewStore() store.UnprocessQuads(aliceIsPerson) } func TestDeleteNonExistingUsedProperty(t *testing.T) { store := NewStore() store.UnprocessQuads(aliceNameAlice) } ================================================ FILE: internal/decompressor/decompressor.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package decompressor import ( "bufio" "bytes" "compress/bzip2" "compress/gzip" "io" ) const ( gzipMagic = "\x1f\x8b" b2zipMagic = "BZh" ) // New detects the file type of an io.Reader between // bzip, gzip, or raw quad file. func New(r io.Reader) (io.Reader, error) { br := bufio.NewReader(r) buf, err := br.Peek(3) if err != nil { return nil, err } switch { case bytes.Equal(buf[:2], []byte(gzipMagic)): return gzip.NewReader(br) case bytes.Equal(buf[:3], []byte(b2zipMagic)): return bzip2.NewReader(br), nil default: return br, nil } } ================================================ FILE: internal/decompressor/decompressor_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package decompressor import ( "bytes" "compress/bzip2" "compress/gzip" "io" "strings" "testing" ) var testDecompressor = []struct { message string input io.Reader expect []byte err error readErr error }{ { message: "text input", input: strings.NewReader("cayley data\n"), err: nil, expect: []byte("cayley data\n"), readErr: nil, }, { message: "gzip input", input: bytes.NewReader([]byte{ 0x1f, 0x8b, 0x08, 0x00, 0x5c, 0xbc, 0xcd, 0x53, 0x00, 0x03, 0x4b, 0x4e, 0xac, 0xcc, 0x49, 0xad, 0x54, 0x48, 0x49, 0x2c, 0x49, 0xe4, 0x02, 0x00, 0x03, 0xe1, 0xfc, 0xc3, 0x0c, 0x00, 0x00, 0x00, }), err: nil, expect: []byte("cayley data\n"), readErr: nil, }, { message: "bzip2 input", input: bytes.NewReader([]byte{ 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xb5, 0x4b, 0xe3, 0xc4, 0x00, 0x00, 0x02, 0xd1, 0x80, 0x00, 0x10, 0x40, 0x00, 0x2e, 0x04, 0x04, 0x20, 0x20, 0x00, 0x31, 0x06, 0x4c, 0x41, 0x4c, 0x1e, 0xa7, 0xa9, 0x2a, 0x18, 0x26, 0xb1, 0xc2, 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x16, 0xa9, 0x7c, 0x78, 0x80, }), err: nil, expect: []byte("cayley data\n"), readErr: nil, }, { message: "bad gzip input", input: strings.NewReader("\x1f\x8bcayley data\n"), err: gzip.ErrHeader, expect: nil, readErr: nil, }, { message: "bad bzip2 input", input: strings.NewReader("\x42\x5a\x68cayley data\n"), err: nil, expect: nil, readErr: bzip2.StructuralError("invalid compression level"), }, } func TestDecompressor(t *testing.T) { for _, test := range testDecompressor { r, err := New(test.input) if err != test.err { t.Fatalf("Unexpected error for %s, got:%v expect:%v", test.message, err, test.err) } if err != nil { continue } p := make([]byte, len(test.expect)*2) n, err := r.Read(p) if err != test.readErr && err != io.EOF { t.Fatalf("Unexpected error for reading %s, got:%v expect:%v", test.message, err, test.err) } if !bytes.Equal(p[:n], test.expect) { t.Errorf("Unexpected read result for %s, got:%q expect:%q", test.message, p[:n], test.expect) } } } ================================================ FILE: internal/dock/dock.go ================================================ //go:build docker // +build docker package dock import ( "fmt" "math/rand" "net" "runtime" "strconv" "testing" "time" docker "github.com/fsouza/go-dockerclient" ) var ( Address = `unix:///var/run/docker.sock` ) type Config struct { docker.Config } type fullConfig struct { docker.Config docker.HostConfig } func run(t testing.TB, conf fullConfig) (addr string) { if testing.Short() { t.SkipNow() } cli, err := docker.NewClient(Address) if err != nil { t.Fatal(err) } // If there is not relevant image at local, pull image from remote repository. if err := cli.PullImage( docker.PullImageOptions{ Repository: conf.Image, }, docker.AuthConfiguration{}, ); err != nil { // If pull image fail, skip the test. t.Skip(err) } cont, err := cli.CreateContainer(docker.CreateContainerOptions{ Config: &conf.Config, HostConfig: &conf.HostConfig, }) if err != nil { t.Skip(err) } t.Cleanup(func() { cli.RemoveContainer(docker.RemoveContainerOptions{ ID: cont.ID, Force: true, }) }) if err := cli.StartContainer(cont.ID, &conf.HostConfig); err != nil { t.Skip(err) } info, err := cli.InspectContainer(cont.ID) if err != nil { t.Skip(err) } addr = info.NetworkSettings.IPAddress return } func randPort() int { const ( min = 10000 max = 30000 ) for { port := min + rand.Intn(max-min) c, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", localhost, port), time.Second) if c != nil { c.Close() } if err != nil { // TODO: check for a specific error return port } } } const localhost = "127.0.0.1" func RunAndWait(t testing.TB, conf Config, port string, check func(string) bool) (addr string) { fconf := fullConfig{Config: conf.Config} if runtime.GOOS != "linux" { lport := strconv.Itoa(randPort()) // nothing except Linux runs Docker natively, // so we randomize the port and expose it on Docker VM fconf.PortBindings = map[docker.Port][]docker.PortBinding{ docker.Port(port + "/tcp"): {{ HostIP: localhost, HostPort: lport, }}, } port = lport } addr = run(t, fconf) if runtime.GOOS != "linux" { // VM ports are automatically exposed on localhost addr = localhost } addr += ":" + port if check == nil { check = waitPort } ok := false for i := 0; i < 10 && !ok; i++ { ok = check(addr) if !ok { time.Sleep(time.Second * 2) } } if !ok { t.Fatal("Container check fails.") } return addr } const wait = time.Second * 5 func waitPort(addr string) bool { start := time.Now() c, err := net.DialTimeout("tcp", addr, wait) if err == nil { c.Close() } else if dt := time.Since(start); dt < wait { time.Sleep(wait - dt) } return err == nil } ================================================ FILE: internal/gephi/stream.go ================================================ package gephi import ( "bytes" "context" "encoding/json" "fmt" "io" "math/rand" "net/http" "strconv" "strings" "github.com/julienschmidt/httprouter" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc/rdf" "github.com/cayleygraph/quad/voc/rdfs" "github.com/cayleygraph/quad/voc/schema" ) const ( defaultLimit = 10000 defaultSize = 20 limitCoord = 500 ) const ( iriInlinePred = quad.IRI("gephi:inline") iriPosX = quad.IRI("gephi:x") iriPosY = quad.IRI("gephi:y") ) var defaultInline = []quad.IRI{ iriPosX, iriPosY, rdf.Type, rdfs.Label, schema.Name, schema.UrlProp, } type GraphStreamHandler struct { QS graph.QuadStore } type valHash [quad.HashSize]byte type GraphStream struct { seen map[valHash]int buf *bytes.Buffer w io.Writer } func printNodeID(id int) string { return strconv.FormatInt(int64(id), 16) } func NewGraphStream(w io.Writer) *GraphStream { return &GraphStream{ w: w, seen: make(map[valHash]int), buf: bytes.NewBuffer(nil), } } func toNodeLabel(v quad.Value) string { if v == nil { return "" } return fmt.Sprint(v.Native()) } func randCoord() float64 { return (rand.Float64() - 0.5) * limitCoord * 2 } func randPos() (x float64, y float64) { x = randCoord() x2 := x * x for y = randCoord(); x2+y*y > limitCoord*limitCoord; y = randCoord() { } return } func setStringProp(v *string, props map[quad.Value]quad.Value, name quad.IRI) { if p, ok := props[name]; ok { if s, ok := p.Native().(string); ok { *v = s } } } func (gs *GraphStream) makeOneNode(id string, v quad.Value, props map[quad.Value]quad.Value) map[string]streamNode { x, y := randPos() var xok, yok bool if p, ok := props[iriPosX]; ok { xok = true switch p := p.(type) { case quad.Int: x = float64(p) case quad.Float: x = float64(p) default: xok = false } } if p, ok := props[iriPosY]; ok { yok = true switch p := p.(type) { case quad.Int: y = float64(p) case quad.Float: y = float64(p) default: yok = false } } var slabel string setStringProp(&slabel, props, rdfs.Label) setStringProp(&slabel, props, schema.Name) var label interface{} if slabel != "" { label = slabel } else { label = v.Native() } node := streamNode{ "label": label, "size": defaultSize, "x": x, "y": y, } for k, v := range props { if k == nil || v == nil || (k == iriPosX && xok) || (k == iriPosY && yok) { continue } node[toNodeLabel(k)] = toNodeLabel(v) } return map[string]streamNode{id: node} } func (gs *GraphStream) AddNode(v quad.Value, props map[quad.Value]quad.Value) string { var h valHash quad.HashTo(v, h[:]) return gs.addNode(v, h, props) } func (gs *GraphStream) encode(o interface{}) { data, _ := json.Marshal(o) gs.buf.Write(data) // Gephi requires \r character at the end of each line gs.buf.WriteString("\r\n") } func (gs *GraphStream) addNode(v quad.Value, h valHash, props map[quad.Value]quad.Value) string { id, ok := gs.seen[h] if ok { return printNodeID(id) } else if v == nil { return "" } id = len(gs.seen) gs.seen[h] = id sid := printNodeID(id) m := gs.makeOneNode(sid, v, props) gs.encode(graphStreamEvent{AddNodes: m}) return sid } func (gs *GraphStream) ChangeNode(v quad.Value, sid string, props map[quad.Value]quad.Value) { m := gs.makeOneNode(sid, v, props) gs.encode(graphStreamEvent{ChangeNodes: m}) } func (gs *GraphStream) AddEdge(i int, s, o string, p quad.Value) { id := "q" + strconv.FormatInt(int64(i), 16) ps := toNodeLabel(p) gs.encode(graphStreamEvent{ AddEdges: map[string]streamEdge{id: { Subject: s, Predicate: ps, Label: ps, Object: o, }}, }) } func (gs *GraphStream) Flush() error { if gs.buf.Len() == 0 { return nil } _, err := gs.buf.WriteTo(gs.w) if err == nil { gs.buf.Reset() } return err } type streamNode map[string]interface{} type streamEdge struct { Subject string `json:"source"` Label string `json:"label"` Predicate string `json:"pred"` Object string `json:"target"` } type graphStreamEvent struct { AddNodes map[string]streamNode `json:"an,omitempty"` ChangeNodes map[string]streamNode `json:"cn,omitempty"` DelNodes map[string]streamNode `json:"dn,omitempty"` AddEdges map[string]streamEdge `json:"ae,omitempty"` ChangeEdges map[string]streamEdge `json:"ce,omitempty"` DelEdges map[string]streamEdge `json:"de,omitempty"` } func (s *GraphStreamHandler) serveRawQuads(ctx context.Context, gs *GraphStream, quads shape.Shape, limit int) { it := shape.BuildIterator(ctx, s.QS, quads).Iterate() defer it.Close() var sh, oh valHash for i := 0; (limit < 0 || i < limit) && it.Next(ctx); i++ { qv := it.Result() if qv == nil { continue } q, err := s.QS.Quad(qv) if err != nil { // TODO: no error handling clog.Warningf("error fetching quad value: %v", err) continue } quad.HashTo(q.Subject, sh[:]) quad.HashTo(q.Object, oh[:]) s, o := gs.addNode(q.Subject, sh, nil), gs.addNode(q.Object, oh, nil) if s == "" || o == "" { continue } gs.AddEdge(i, s, o, q.Predicate) if err := gs.Flush(); err != nil { return } } } func shouldInline(v quad.Value) bool { switch v.(type) { case quad.Bool, quad.Int, quad.Float: return true } return false } func (s *GraphStreamHandler) serveNodesWithProps(ctx context.Context, gs *GraphStream, limit int) { propsPath := path.NewPath(s.QS).Has(iriInlinePred, quad.Bool(true)) // list of predicates marked as inline properties for gephi inline := make(map[quad.Value]struct{}) err := propsPath.Iterate(ctx).EachValue(s.QS, func(v quad.Value) error { inline[v] = struct{}{} return nil }) if err != nil { clog.Errorf("cannot iterate over properties: %v", err) return } // inline some well-known predicates for _, iri := range defaultInline { inline[iri] = struct{}{} inline[iri.Full()] = struct{}{} } ignore := make(map[quad.Value]struct{}) nodes := iterator.NewNot(propsPath.BuildIterator(ctx), s.QS.NodesAllIterator()) ictx, cancel := context.WithCancel(ctx) defer cancel() itc := iterator.Iterate(ictx, nodes).On(s.QS).Limit(limit) qi := 0 _ = itc.EachValuePair(s.QS, func(v graph.Ref, nv quad.Value) error { if _, skip := ignore[nv]; skip { return nil } // list of inline properties props := make(map[quad.Value]quad.Value) var ( sid string h, oh valHash ) quad.HashTo(nv, h[:]) predIt := s.QS.QuadIterator(quad.Subject, v).Iterate() defer predIt.Close() for predIt.Next(ictx) { // this check helps us ignore nodes with no links if sid == "" { sid = gs.addNode(nv, h, props) } q, err := s.QS.Quad(predIt.Result()) if err != nil { continue } if _, ok := inline[q.Predicate]; ok { props[q.Predicate] = q.Object ignore[q.Object] = struct{}{} } else if shouldInline(q.Object) { props[q.Predicate] = q.Object } else { quad.HashTo(q.Object, oh[:]) o := gs.addNode(q.Object, oh, nil) if o == "" { continue } gs.AddEdge(qi, sid, o, q.Predicate) qi++ if err := gs.Flush(); err != nil { cancel() return nil } } } if err := predIt.Err(); err != nil { cancel() return nil } else if sid == "" { return nil } if len(props) != 0 { gs.ChangeNode(nv, sid, props) } if err := gs.Flush(); err != nil { cancel() } return nil }) } func valuesFromString(s string) []quad.Value { if s == "" { return nil } arr := strings.Split(s, ",") out := make([]quad.Value, 0, len(arr)) for _, s := range arr { out = append(out, quad.StringToValue(s)) } return out } func (s *GraphStreamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { ctx := context.TODO() var limit int if s := r.FormValue("limit"); s != "" { limit, _ = strconv.Atoi(s) } if limit == 0 { limit = defaultLimit } mode := "raw" if s := r.FormValue("mode"); s != "" { mode = s } w.Header().Set("Content-Type", "application/stream+json") gs := NewGraphStream(w) switch mode { case "nodes": s.serveNodesWithProps(ctx, gs, limit) case "raw": values := shape.FilterQuads( valuesFromString(r.FormValue("sub")), valuesFromString(r.FormValue("pred")), valuesFromString(r.FormValue("obj")), valuesFromString(r.FormValue("label")), ) s.serveRawQuads(ctx, gs, values, limit) default: w.WriteHeader(http.StatusBadRequest) return } } ================================================ FILE: internal/gephi/stream_test.go ================================================ package gephi import ( "bytes" "testing" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" ) func TestStreamEncoder(t *testing.T) { buf := bytes.NewBuffer(nil) gs := NewGraphStream(buf) p := map[quad.Value]quad.Value{iriPosX: quad.Float(0), iriPosY: quad.Float(0)} gs.AddNode(quad.String("aaa"), p) gs.AddNode(quad.String("bbb"), p) gs.Flush() const expect = "{\"an\":{\"0\":{\"label\":\"aaa\",\"size\":20,\"x\":0,\"y\":0}}}\r\n{\"an\":{\"1\":{\"label\":\"bbb\",\"size\":20,\"x\":0,\"y\":0}}}\r\n" require.Equal(t, expect, buf.String()) } ================================================ FILE: internal/http/api_v1.go ================================================ package http import ( "net/http" "github.com/cayleygraph/cayley/graph" cayleyhttp "github.com/cayleygraph/cayley/server/http" "github.com/julienschmidt/httprouter" ) type API struct { config *Config handle *graph.Handle } func (api *API) GetHandleForRequest(r *http.Request) (*graph.Handle, error) { return cayleyhttp.HandleForRequest(api.handle, "single", nil, r) } func (api *API) RWOnly(handler httprouter.Handle) httprouter.Handle { if api.config.ReadOnly { return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { jsonResponse(w, http.StatusForbidden, "Database is read-only.") } } return handler } func (api *API) APIv1(r *httprouter.Router) { r.POST("/query/:query_lang", api.ServeV1Query) r.POST("/shape/:query_lang", api.ServeV1Shape) r.POST("/write", api.RWOnly(api.ServeV1Write)) r.POST("/write/file/nquad", api.RWOnly(api.ServeV1WriteNQuad)) r.POST("/delete", api.RWOnly(api.ServeV1Delete)) } ================================================ FILE: internal/http/cors.go ================================================ package http import ( "net/http" ) // CORS adds CORS related headers to responses func CORS(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if origin := req.Header.Get("Origin"); origin != "" { w.Header().Set("Access-Control-Allow-Origin", origin) w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization") } h.ServeHTTP(w, req) }) } // HandlePreflight is an http.Handler for CORS Prelight requests func HandlePreflight(w http.ResponseWriter, r *http.Request) { // Adjust status code to 204 w.WriteHeader(http.StatusNoContent) } ================================================ FILE: internal/http/health.go ================================================ package http import "net/http" // HandleHealth is a route for handling health checks to the server func HandleHealth(w http.ResponseWriter, r *http.Request) { // Adjust status code to 204 w.WriteHeader(http.StatusNoContent) } ================================================ FILE: internal/http/http.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package http import ( "encoding/json" "fmt" "io/fs" "net/http" "time" "github.com/julienschmidt/httprouter" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/internal/gephi" cayleyhttp "github.com/cayleygraph/cayley/server/http" "github.com/cayleygraph/cayley/ui" ) func jsonResponse(w http.ResponseWriter, code int, err interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) w.Write([]byte(`{"error": `)) data, _ := json.Marshal(fmt.Sprint(err)) w.Write(data) w.Write([]byte(`}`)) } // Config holds the HTTP server configuration type Config struct { ReadOnly bool Timeout time.Duration Batch int } func SetupRoutes(handle *graph.Handle, cfg *Config) error { ui, err := fs.Sub(ui.FS, "web") if err != nil { return err } r := httprouter.New() // Health check r.HandlerFunc("GET", "/health", HandleHealth) // Handle CORS preflight request r.HandlerFunc("OPTIONS", "/*path", HandlePreflight) // Register API V1 api := &API{config: cfg, handle: handle} api.APIv1(r) // Register Gephi API gs := &gephi.GraphStreamHandler{QS: handle.QuadStore} r.GET("/gephi/gs", gs.ServeHTTP) // Register API V2 api2 := cayleyhttp.NewBoundAPIv2(handle, r) api2.SetReadOnly(cfg.ReadOnly) api2.SetBatchSize(cfg.Batch) api2.SetQueryTimeout(cfg.Timeout) // For non API requests serve the UI r.NotFound = http.FileServer(http.FS(ui)) http.Handle("/", CORS(LogRequest(r))) return nil } ================================================ FILE: internal/http/http_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package http import ( "fmt" "reflect" "testing" "github.com/cayleygraph/quad" ) var parseTests = []struct { message string input string expect []quad.Quad err error }{ { message: "parse correct JSON", input: `[ {"subject": "foo", "predicate": "bar", "object": "baz"}, {"subject": "foo", "predicate": "bar", "object": "baz", "label": "graph"} ]`, expect: []quad.Quad{ quad.Make("foo", "bar", "baz", nil), quad.Make("foo", "bar", "baz", "graph"), }, err: nil, }, { message: "parse correct JSON with extra field", input: `[ {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} ]`, expect: []quad.Quad{ quad.Make("foo", "bar", "foo", nil), }, err: nil, }, { message: "reject incorrect JSON", input: `[ {"subject": "foo", "predicate": "bar"} ]`, expect: nil, err: fmt.Errorf("invalid quad at index %d. %v", 0, quad.Make("foo", "bar", nil, nil)), }, } func TestParseJSON(t *testing.T) { for _, test := range parseTests { got, err := ParseJSONToQuadList([]byte(test.input)) if fmt.Sprint(err) != fmt.Sprint(test.err) { t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err) } if !reflect.DeepEqual(got, test.expect) { t.Errorf("Failed to %v, got:%v expect:%v", test.message, got, test.expect) } } } ================================================ FILE: internal/http/logs.go ================================================ package http import ( "net/http" "time" "github.com/cayleygraph/cayley/clog" ) // statusWriter wraps http.ResponseWriter and captures the written status code type statusWriter struct { http.ResponseWriter code int } // newStatusWriter returns an initialized statusWriter func newStatusWriter(w http.ResponseWriter) *statusWriter { return &statusWriter{w, 200} } // WriteHeader wraps ResponseWriter WriteHeader and saves the code func (w *statusWriter) WriteHeader(code int) { w.ResponseWriter.WriteHeader(code) w.code = code } // getAddress returns the address of the incoming request func getAddress(req *http.Request) string { addr := req.Header.Get("X-Real-IP") if addr == "" { addr = req.Header.Get("X-Forwarded-For") if addr == "" { addr = req.RemoteAddr } } return addr } // LogRequest wraps a http.Handler and emits logs about the request and the response func LogRequest(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { start := time.Now() addr := getAddress(req) sw := newStatusWriter(w) clog.Infof("started %s %s for %s", req.Method, req.URL.Path, addr) handler.ServeHTTP(sw, req) clog.Infof("completed %v %s %s in %v", sw.code, http.StatusText(sw.code), req.URL.Path, time.Since(start)) }) } ================================================ FILE: internal/http/query.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package http import ( "context" "encoding/json" "errors" "io" "io/ioutil" "net/http" "net/url" "strconv" "github.com/julienschmidt/httprouter" "github.com/cayleygraph/cayley/query" ) type SuccessQueryWrapper struct { Result interface{} `json:"result"` } type ErrorQueryWrapper struct { Error string `json:"error"` } func WriteError(w io.Writer, err error) error { enc := json.NewEncoder(w) //enc.SetIndent("", " ") return enc.Encode(ErrorQueryWrapper{err.Error()}) } func WriteResult(w io.Writer, result interface{}) error { enc := json.NewEncoder(w) //enc.SetIndent("", " ") return enc.Encode(SuccessQueryWrapper{result}) } func (api *API) contextForRequest(r *http.Request) (context.Context, func()) { ctx := context.TODO() // TODO(dennwc): get from request cancel := func() {} if api.config.Timeout > 0 { ctx, cancel = context.WithTimeout(ctx, api.config.Timeout) } return ctx, cancel } func defaultErrorFunc(w query.ResponseWriter, err error) { data, _ := json.Marshal(err.Error()) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(`{"error" : `)) w.Write(data) w.Write([]byte(`}`)) } // TODO(barakmich): Turn this into proper middleware. // ServeV1Query is the HTTP handler for queries in API V1 func (api *API) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) { ctx, cancel := api.contextForRequest(r) defer cancel() l := query.GetLanguage(params.ByName("query_lang")) if l == nil { jsonResponse(w, http.StatusBadRequest, "Unknown query language.") return } errFunc := defaultErrorFunc if l.HTTPError != nil { errFunc = l.HTTPError } select { case <-ctx.Done(): errFunc(w, ctx.Err()) return default: } h, err := api.GetHandleForRequest(r) if err != nil { errFunc(w, err) return } if l.HTTPQuery != nil { defer r.Body.Close() l.HTTPQuery(ctx, h.QuadStore, w, r.Body) return } if l.Session == nil { errFunc(w, errors.New("no support for HTTP interface for this query language")) return } par, _ := url.ParseQuery(r.URL.RawQuery) limit, _ := strconv.Atoi(par.Get("limit")) if limit == 0 { limit = 100 } ses := l.Session(h.QuadStore) bodyBytes, err := ioutil.ReadAll(r.Body) if err != nil { errFunc(w, err) return } it, err := ses.Execute(ctx, string(bodyBytes), query.Options{ Collation: query.JSON, Limit: limit, }) if err != nil { errFunc(w, err) return } defer it.Close() var out []interface{} for it.Next(ctx) { out = append(out, it.Result()) } if err = it.Err(); err != nil { errFunc(w, err) return } _ = WriteResult(w, out) } func (api *API) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) { jsonResponse(w, http.StatusNotImplemented, "Query shape API v1 is deprecated.") } ================================================ FILE: internal/http/write.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package http import ( "encoding/json" "errors" "fmt" "io" "io/ioutil" "net/http" "strconv" "github.com/julienschmidt/httprouter" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/internal/decompressor" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/nquads" ) func ParseJSONToQuadList(jsonBody []byte) (out []quad.Quad, _ error) { var quads []struct { Subject string `json:"subject"` Predicate string `json:"predicate"` Object string `json:"object"` Label string `json:"label"` } err := json.Unmarshal(jsonBody, &quads) if err != nil { return nil, err } out = make([]quad.Quad, 0, len(quads)) for i, jq := range quads { q := quad.Quad{ Subject: quad.StringToValue(jq.Subject), Predicate: quad.StringToValue(jq.Predicate), Object: quad.StringToValue(jq.Object), Label: quad.StringToValue(jq.Label), } if !q.IsValid() { return nil, fmt.Errorf("invalid quad at index %d. %s", i, q) } out = append(out, q) } return out, nil } const maxQuerySize = 1024 * 1024 // 1 MB func readLimit(r io.Reader) ([]byte, error) { lr := io.LimitReader(r, maxQuerySize).(*io.LimitedReader) data, err := ioutil.ReadAll(lr) if err != nil && lr.N <= 0 { err = errors.New("request is too large") } return data, err } func (api *API) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if api.config.ReadOnly { jsonResponse(w, 400, "Database is read-only.") return } // TODO: streaming reader bodyBytes, err := readLimit(r.Body) if err != nil { jsonResponse(w, 400, err) return } quads, err := ParseJSONToQuadList(bodyBytes) if err != nil { jsonResponse(w, 400, err) return } h, err := api.GetHandleForRequest(r) if err != nil { jsonResponse(w, 400, err) return } if err = h.QuadWriter.AddQuadSet(quads); err != nil { jsonResponse(w, 400, err) return } fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", len(quads)) } func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) { if api.config.ReadOnly { jsonResponse(w, 400, "Database is read-only.") return } formFile, _, err := r.FormFile("NQuadFile") if err != nil { clog.Errorf("%v", err) jsonResponse(w, 500, "Couldn't read file: "+err.Error()) return } defer formFile.Close() blockSize, blockErr := strconv.Atoi(r.URL.Query().Get("block_size")) if blockErr != nil { blockSize = quad.DefaultBatch } quadReader, err := decompressor.New(formFile) // TODO(kortschak) Make this configurable from the web UI. dec := nquads.NewReader(quadReader, false) h, err := api.GetHandleForRequest(r) if err != nil { jsonResponse(w, 400, err) return } qw := graph.NewWriter(h.QuadWriter) n, err := quad.CopyBatch(qw, dec, blockSize) if err != nil { jsonResponse(w, 400, err) return } err = qw.Close() if err != nil { jsonResponse(w, 400, err) return } fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n) } func (api *API) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) { if api.config.ReadOnly { jsonResponse(w, 400, "Database is read-only.") return } bodyBytes, err := readLimit(r.Body) if err != nil { jsonResponse(w, 400, err) return } quads, err := ParseJSONToQuadList(bodyBytes) if err != nil { jsonResponse(w, 400, err) return } h, err := api.GetHandleForRequest(r) if err != nil { jsonResponse(w, 400, err) return } for _, q := range quads { err = h.QuadWriter.RemoveQuad(q) if err != nil && !graph.IsQuadNotExist(err) { jsonResponse(w, 400, err) return } } fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d quads.\"}", len(quads)) } ================================================ FILE: internal/linkedql/schema/schema.go ================================================ package schema import ( "encoding/json" "reflect" "strconv" "github.com/cayleygraph/cayley/query/linkedql" // Steps are imported here so they be registered and documented in the schema _ "github.com/cayleygraph/cayley/query/linkedql/steps" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc/owl" "github.com/cayleygraph/quad/voc/rdf" "github.com/cayleygraph/quad/voc/rdfs" "github.com/cayleygraph/quad/voc/xsd" ) // rdfgGraph is the W3C type for named graphs const ( rdfgNamespace = "http://www.w3.org/2004/03/trix/rdfg-1/" rdfgPrefix = "rdfg:" rdfgGraph = rdfgPrefix + "Graph" ) var ( pathStep = reflect.TypeOf((*linkedql.PathStep)(nil)).Elem() iteratorStep = reflect.TypeOf((*linkedql.IteratorStep)(nil)).Elem() entityIdentifier = reflect.TypeOf((*linkedql.EntityIdentifier)(nil)).Elem() value = reflect.TypeOf((*quad.Value)(nil)).Elem() propertyPath = reflect.TypeOf((*linkedql.PropertyPath)(nil)) stringMap = reflect.TypeOf(map[string]string{}) graphPattern = reflect.TypeOf(linkedql.GraphPattern(nil)) ) func typeToRange(t reflect.Type) string { if t == stringMap { return "rdf:JSON" } if t == graphPattern { return rdfgGraph } if t.Kind() == reflect.Slice { return typeToRange(t.Elem()) } if t.Kind() == reflect.String { return xsd.String } if t.Kind() == reflect.Bool { return xsd.Boolean } if kind := t.Kind(); kind == reflect.Int64 || kind == reflect.Int { return xsd.Int } if t.Implements(value) { return rdfs.Resource } if t == entityIdentifier { return owl.Thing } if t == pathStep { return linkedql.Prefix + "PathStep" } if t == propertyPath { return linkedql.Prefix + "PropertyPath" } panic("Unexpected type " + t.String()) } // identified is used for referencing a type type identified struct { ID string `json:"@id"` } // newIdentified creates new identified struct func newIdentified(id string) identified { return identified{ID: id} } // cardinalityRestriction is used to indicate a how many values can a property get type cardinalityRestriction struct { ID string `json:"@id"` Type string `json:"@type"` Cardinality int `json:"owl:cardinality"` Property identified `json:"owl:onProperty"` } func newBlankNodeID() string { return quad.RandomBlankNode().String() } // newSingleCardinalityRestriction creates a cardinality of 1 restriction for given property func newSingleCardinalityRestriction(prop string) cardinalityRestriction { return cardinalityRestriction{ ID: newBlankNodeID(), Type: owl.Restriction, Cardinality: 1, Property: identified{ID: prop}, } } type owlPropertyRestriction struct { ID string `json:"@id"` Type string `json:"@type"` Property identified `json:"owl:onProperty"` } func newOWLPropertyRestriction(prop string) owlPropertyRestriction { return owlPropertyRestriction{ ID: newBlankNodeID(), Type: owl.Restriction, Property: identified{ID: prop}, } } // minCardinalityRestriction is used to indicate a how many values can a property get at the very least type minCardinalityRestriction struct { owlPropertyRestriction MinCardinality int `json:"owl:minCardinality"` } // maxCardinalityRestriction is used to indicate a how many values can a property get at most type maxCardinalityRestriction struct { owlPropertyRestriction MaxCardinality int `json:"owl:maxCardinality"` } func newMinCardinalityRestriction(prop string, minCardinality int) minCardinalityRestriction { return minCardinalityRestriction{ owlPropertyRestriction: newOWLPropertyRestriction(prop), MinCardinality: minCardinality, } } func newSingleMaxCardinalityRestriction(prop string) maxCardinalityRestriction { return maxCardinalityRestriction{ owlPropertyRestriction: newOWLPropertyRestriction(prop), MaxCardinality: 1, } } // getOWLPropertyType for given kind of value type returns property OWL type func getOWLPropertyType(kind reflect.Kind) string { if kind == reflect.String || kind == reflect.Bool || kind == reflect.Int64 || kind == reflect.Int { return owl.DatatypeProperty } return owl.ObjectProperty } // property is used to declare a property type property struct { ID string `json:"@id"` Type string `json:"@type"` Domain interface{} `json:"rdfs:domain"` Range interface{} `json:"rdfs:range"` } // class is used to declare a class type class struct { ID string `json:"@id"` Type string `json:"@type"` Comment string `json:"rdfs:comment"` SuperClasses []interface{} `json:"rdfs:subClassOf"` } // newClass creates a new class struct func newClass(id string, superClasses []interface{}, comment string) class { return class{ ID: id, Type: rdfs.Class, SuperClasses: superClasses, Comment: comment, } } // getStepTypeClasses for given step type returns the matching class identifiers func getStepTypeClasses(t reflect.Type) []string { var typeClasses []string if t.Implements(pathStep) { typeClasses = append(typeClasses, linkedql.Prefix+"PathStep") } if t.Implements(iteratorStep) { typeClasses = append(typeClasses, linkedql.Prefix+"IteratorStep") } return typeClasses } type list struct { Members []interface{} `json:"@list"` } func newList(members []interface{}) list { return list{ Members: members, } } type unionOf struct { ID string `json:"@id"` Type string `json:"@type"` List list `json:"owl:unionOf"` } func newUnionOf(classes []string) unionOf { var members []interface{} for _, class := range classes { members = append(members, newIdentified(class)) } return unionOf{ ID: newBlankNodeID(), Type: owl.Class, List: newList(members), } } func newGenerator() *generator { return &generator{ propToTypes: make(map[string]map[string]struct{}), propToDomains: make(map[string]map[string]struct{}), propToRanges: make(map[string]map[string]struct{}), } } type generator struct { out []interface{} propToTypes map[string]map[string]struct{} propToDomains map[string]map[string]struct{} propToRanges map[string]map[string]struct{} } // returns super types func (g *generator) addTypeFields(name string, t reflect.Type, indirect bool) []interface{} { var super []interface{} for j := 0; j < t.NumField(); j++ { f := t.Field(j) if f.Anonymous { if f.Type.Kind() != reflect.Struct || !indirect { continue } super = append(super, g.addTypeFields(name, f.Type, false)...) continue } prop := linkedql.Prefix + f.Tag.Get("json") var hasMinCardinality bool v, ok := f.Tag.Lookup("minCardinality") if ok { minCardinality, err := strconv.Atoi(v) if err != nil { panic(err) } hasMinCardinality = true super = append(super, newMinCardinalityRestriction(prop, minCardinality)) } if f.Type.Kind() != reflect.Slice { if hasMinCardinality { super = append(super, newSingleMaxCardinalityRestriction(prop)) } else { super = append(super, newSingleCardinalityRestriction(prop)) } } typ := getOWLPropertyType(f.Type.Kind()) if g.propToTypes[prop] == nil { g.propToTypes[prop] = make(map[string]struct{}) } g.propToTypes[prop][typ] = struct{}{} if g.propToDomains[prop] == nil { g.propToDomains[prop] = make(map[string]struct{}) } g.propToDomains[prop][name] = struct{}{} if g.propToRanges[prop] == nil { g.propToRanges[prop] = make(map[string]struct{}) } g.propToRanges[prop][typeToRange(f.Type)] = struct{}{} } return super } func (g *generator) AddType(name string, t reflect.Type) { step, ok := reflect.New(t).Interface().(linkedql.Step) if !ok { return } var super []interface{} stepTypeClasses := getStepTypeClasses(reflect.PtrTo(t)) for _, typeClass := range stepTypeClasses { super = append(super, newIdentified(typeClass)) } super = append(super, g.addTypeFields(name, t, true)...) g.out = append(g.out, newClass(name, super, step.Description())) } func (g *generator) Generate() []byte { for prop, types := range g.propToTypes { if len(types) != 1 { panic("Properties must be either object properties or datatype properties. " + prop + " has both.") } var typ string for t := range types { typ = t break } var domains []string for d := range g.propToDomains[prop] { domains = append(domains, d) } var ranges []string for r := range g.propToRanges[prop] { ranges = append(ranges, r) } var dom interface{} if len(domains) == 1 { dom = identified{domains[0]} } else { dom = newUnionOf(domains) } var rng interface{} if len(ranges) == 1 { rng = newIdentified(ranges[0]) } else { rng = newUnionOf(ranges) } g.out = append(g.out, property{ ID: prop, Type: typ, Domain: dom, Range: rng, }) } graph := []interface{}{ map[string]string{ "@id": linkedql.Prefix + "Step", "@type": owl.Class, }, map[string]interface{}{ "@id": linkedql.Prefix + "PathStep", "@type": owl.Class, rdfs.SubClassOf: identified{ID: linkedql.Prefix + "Step"}, }, map[string]interface{}{ "@id": linkedql.Prefix + "IteratorStep", "@type": owl.Class, rdfs.SubClassOf: identified{ID: linkedql.Prefix + "Step"}, }, } graph = append(graph, g.out...) data, err := json.Marshal(map[string]interface{}{ "@context": map[string]interface{}{ "rdf": rdf.NS, "rdfs": rdfs.NS, "owl": owl.NS, "xsd": xsd.NS, "linkedql": linkedql.Namespace, "rdfg": rdfgNamespace, }, "@graph": graph, }) if err != nil { panic(err) } return data } // Generate a schema in JSON-LD format that contains all registered LinkedQL types and properties. func Generate() []byte { g := newGenerator() for _, name := range linkedql.RegisteredTypes() { t, ok := linkedql.TypeByName(name) if !ok { panic("type is registered, but the lookup fails") } g.AddType(name, t) } return g.Generate() } ================================================ FILE: internal/linkedql/schema/schema_test.go ================================================ package schema import ( "encoding/json" "testing" ) func TestMarshalSchema(t *testing.T) { out := Generate() var o interface{} err := json.Unmarshal(out, &o) if err != nil { t.Fatal(err) } } ================================================ FILE: internal/load.go ================================================ package internal import ( "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strings" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/internal/decompressor" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/nquads" ) // Load loads a graph from the given path and write it to qw. See // DecompressAndLoad for more information. func Load(qw quad.WriteCloser, batch int, path, typ string) error { return DecompressAndLoad(qw, batch, path, typ) } type readCloser struct { quad.ReadCloser close func() error } func (r readCloser) Close() error { err := r.ReadCloser.Close() if r.close != nil { r.close() } return err } type nopCloser struct { quad.Reader } func (r nopCloser) Close() error { return nil } func QuadReaderFor(path, typ string) (quad.ReadCloser, error) { var ( r io.Reader c io.Closer ) if path == "-" { r = os.Stdin } else if u, err := url.Parse(path); err != nil || u.Scheme == "file" || u.Scheme == "" { // Don't alter relative URL path or non-URL path parameter. if u.Scheme != "" && err == nil { // Recovery heuristic for mistyping "file://path/to/file". path = filepath.Join(u.Host, u.Path) } f, err := os.Open(path) if os.IsNotExist(err) { return nil, err } else if err != nil { return nil, fmt.Errorf("could not open file %q: %v", path, err) } r, c = f, f } else { res, err := http.Get(path) if err != nil { return nil, fmt.Errorf("could not get resource <%s>: %v", u, err) } // TODO(dennwc): save content type for format auto-detection r, c = res.Body, res.Body } r, err := decompressor.New(r) if err != nil { if c != nil { c.Close() } if err == io.EOF { return nopCloser{quad.NewReader(nil)}, nil } return nil, err } var qr quad.ReadCloser switch typ { case "cquad", "nquad": // legacy qr = nquads.NewReader(r, false) default: var format *quad.Format if typ == "" { name := filepath.Base(path) name = strings.TrimSuffix(name, ".gz") name = strings.TrimSuffix(name, ".bz2") format = quad.FormatByExt(filepath.Ext(name)) if format == nil { typ = "nquads" } } if format == nil { format = quad.FormatByName(typ) } if format == nil { err = fmt.Errorf("unknown quad format %q", typ) } else if format.Reader == nil { err = fmt.Errorf("decoding of %q is not supported", typ) } if err != nil { if c != nil { c.Close() } return nil, err } qr = format.Reader(r) } if c != nil { return readCloser{ReadCloser: qr, close: c.Close}, nil } return qr, nil } // DecompressAndLoad will load or fetch a graph from the given path, decompress // it, and then call the given load function to process the decompressed graph. // If no loadFn is provided, db.Load is called. func DecompressAndLoad(qw quad.WriteCloser, batch int, path, typ string) error { if path == "" { return nil } qr, err := QuadReaderFor(path, typ) if err != nil { return err } defer qr.Close() _, err = quad.CopyBatch(&batchLogger{w: qw}, qr, batch) if err != nil { return fmt.Errorf("db: failed to load data: %v", err) } return qw.Close() } type batchLogger struct { cnt int w quad.Writer } func (w *batchLogger) WriteQuads(quads []quad.Quad) (int, error) { n, err := w.w.WriteQuads(quads) if clog.V(2) { w.cnt += n clog.Infof("Wrote %d quads.", w.cnt) } return n, err } ================================================ FILE: internal/lru/lru.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lru import ( "container/list" "sync" ) // TODO(kortschak) Reimplement without container/list. // Cache implements an LRU cache. type Cache struct { mu sync.Mutex cache map[string]*list.Element priority *list.List maxSize int } type kv struct { key string value interface{} } func New(size int) *Cache { return &Cache{ maxSize: size, priority: list.New(), cache: make(map[string]*list.Element), } } func (lru *Cache) Put(key string, value interface{}) { if _, ok := lru.Get(key); ok { return } lru.mu.Lock() defer lru.mu.Unlock() if len(lru.cache) == lru.maxSize { last := lru.priority.Remove(lru.priority.Back()) delete(lru.cache, last.(kv).key) } lru.priority.PushFront(kv{key: key, value: value}) lru.cache[key] = lru.priority.Front() } func (lru *Cache) Del(key string) { lru.mu.Lock() defer lru.mu.Unlock() e := lru.cache[key] if e == nil { return } delete(lru.cache, key) lru.priority.Remove(e) } func (lru *Cache) Get(key string) (interface{}, bool) { lru.mu.Lock() defer lru.mu.Unlock() if element, ok := lru.cache[key]; ok { lru.priority.MoveToFront(element) return element.Value.(kv).value, true } return nil, false } ================================================ FILE: internal/lru/lru_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lru import ( "fmt" "testing" ) // checks fatal error: concurrent map read and map write func TestPanicLRUCache(t *testing.T) { xch := make(chan int) c := New(1024) for i := 0; i < 100; i++ { go func(i int) { key := fmt.Sprintf("Key%d", i) c.Put(key, i) c.Get(key) xch <- i }(i) } for i := 0; i < 100; i++ { <-xch } } ================================================ FILE: internal/repl/repl.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build !appengine package repl import ( "context" "fmt" "io" "os" "os/signal" "strconv" "strings" "time" "github.com/peterh/liner" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/quad/nquads" ) func trace(s string) (string, time.Time) { return s, time.Now() } func un(s string, startTime time.Time) { endTime := time.Now() fmt.Printf(s, float64(endTime.UnixNano()-startTime.UnixNano())/float64(1e6)) } func Run(ctx context.Context, qu string, ses query.REPLSession) error { nResults := 0 startTrace, startTime := trace("Elapsed time: %g ms\n\n") defer func() { if nResults > 0 { un(startTrace, startTime) } }() fmt.Printf("\n") it, err := ses.Execute(ctx, qu, query.Options{ Collation: query.REPL, Limit: 100, }) if err != nil { return err } defer it.Close() for it.Next(ctx) { fmt.Print(it.Result()) nResults++ } if err := it.Err(); err != nil { return err } if nResults > 0 { results := "Result" if nResults > 1 { results += "s" } fmt.Printf("-----------\n%d %s\n", nResults, results) } return nil } const ( defaultLanguage = "gizmo" ps1 = "cayley> " ps2 = "... " history = ".cayley_history" ) func Repl(ctx context.Context, h *graph.Handle, queryLanguage string, timeout time.Duration) error { if queryLanguage == "" { queryLanguage = defaultLanguage } l := query.GetLanguage(queryLanguage) if l == nil || l.Session == nil { return fmt.Errorf("unsupported query language: %q", queryLanguage) } ses := l.Session(h.QuadStore) term, err := terminal(history) if os.IsNotExist(err) { fmt.Printf("creating new history file: %q\n", history) } defer persist(term, history) var ( prompt = ps1 code string ) newCtx := func() (context.Context, func()) { return ctx, func() {} } if timeout > 0 { newCtx = func() (context.Context, func()) { return context.WithTimeout(ctx, timeout) } } for { select { case <-ctx.Done(): return ctx.Err() default: } if len(code) == 0 { prompt = ps1 } else { prompt = ps2 } line, err := term.Prompt(prompt) if err != nil { if err == io.EOF { fmt.Println() return nil } return err } term.AppendHistory(line) line = strings.TrimSpace(line) if len(line) == 0 || line[0] == '#' { continue } if code == "" { cmd, args := splitLine(line) switch cmd { case ":debug": args = strings.TrimSpace(args) var debug bool switch args { case "t": debug = true case "f": // Do nothing. default: debug, err = strconv.ParseBool(args) if err != nil { fmt.Printf("Error: cannot parse %q as a valid boolean - acceptable values: 't'|'true' or 'f'|'false'\n", args) continue } } if debug { clog.SetV(2) } else { clog.SetV(0) } fmt.Printf("Debug set to %t\n", debug) continue case ":a": quad, err := nquads.Parse(args) if err == nil { err = h.QuadWriter.AddQuad(quad) } if err != nil { fmt.Printf("Error: not a valid quad: %v\n", err) continue } continue case ":d": quad, err := nquads.Parse(args) if err != nil { fmt.Printf("Error: not a valid quad: %v\n", err) continue } err = h.QuadWriter.RemoveQuad(quad) if err != nil { fmt.Printf("error deleting: %v\n", err) } continue case "help": fmt.Printf("Help\n\texit // Exit\n\thelp // this help\n\td: // delete quad\n\ta: // add quad\n\t:debug [t|f]\n") continue case "exit": term.Close() os.Exit(0) default: if cmd[0] == ':' { fmt.Printf("Unknown command: %q\n", cmd) continue } } } code += line nctx, cancel := newCtx() err = Run(nctx, code, ses) cancel() if err == query.ErrParseMore { // collect more input } else if err != nil { fmt.Println("Error: ", err) code = "" } else { code = "" } } } // Splits a line into a command and its arguments // e.g. ":a b c d ." will be split into ":a" and " b c d ." func splitLine(line string) (string, string) { var command, arguments string line = strings.TrimSpace(line) // An empty line/a line consisting of whitespace contains neither command nor arguments if len(line) > 0 { command = strings.Fields(line)[0] // A line containing only a command has no arguments if len(line) > len(command) { arguments = line[len(command):] } } return command, arguments } func terminal(path string) (*liner.State, error) { term := liner.NewLiner() go func() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill) <-c err := persist(term, history) if err != nil { fmt.Fprintf(os.Stderr, "failed to properly clean up terminal: %v\n", err) os.Exit(1) } os.Exit(0) }() f, err := os.Open(path) if err != nil { return term, err } defer f.Close() _, err = term.ReadHistory(f) return term, err } func persist(term *liner.State, path string) error { f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) if err != nil { return fmt.Errorf("could not open %q to append history: %v", path, err) } defer f.Close() _, err = term.WriteHistory(f) if err != nil { return fmt.Errorf("could not write history to %q: %v", path, err) } return term.Close() } ================================================ FILE: internal/repl/repl_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package repl import ( "testing" ) var testSplitLines = []struct { line string expectedCommand string expectedArguments string err error }{ { line: ":a arg1 arg2 arg3 .", expectedCommand: ":a", expectedArguments: " arg1 arg2 arg3 .", }, { line: ":debug t", expectedCommand: ":debug", expectedArguments: " t", }, { line: "", // expectedCommand is nil // expectedArguments is nil }, { line: `:d . # comments here`, expectedCommand: ":d", expectedArguments: ` . # comments here`, }, { line: ` :a subject "predicate with spaces" object . `, expectedCommand: ":a", expectedArguments: ` subject "predicate with spaces" object .`, }, } func TestSplitLines(t *testing.T) { for _, testcase := range testSplitLines { command, arguments := splitLine(testcase.line) if testcase.expectedCommand != command { t.Errorf("Error splitting lines: got: %v expected: %v", command, testcase.expectedCommand) } if testcase.expectedArguments != arguments { t.Errorf("Error splitting lines: got: %v expected: %v", arguments, testcase.expectedArguments) } } } ================================================ FILE: query/gizmo/environ.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gizmo // Builds a new Gizmo environment pointing at a session. import ( "fmt" "regexp" "time" "github.com/dop251/goja" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) // graphObject is a root graph object. // // Name: `graph`, Alias: `g` // // This is the only special object in the environment, generates the query objects. // Under the hood, they're simple objects that get compiled to a Go iterator tree when executed. // Methods starting with "New" are accessible in JavaScript with a capital letter (e.g. NewV becomes V) type graphObject struct { s *Session } // Uri creates an IRI values from a given string. func (g *graphObject) NewIRI(s string) quad.IRI { return quad.IRI(g.s.ns.FullIRI(s)) } // AddNamespace associates prefix with a given IRI namespace. func (g *graphObject) AddNamespace(pref, ns string) { g.s.ns.Register(voc.Namespace{Prefix: pref + ":", Full: ns}) } // AddDefaultNamespaces register all default namespaces for automatic IRI resolution. func (g *graphObject) AddDefaultNamespaces() { voc.CloneTo(&g.s.ns) } // LoadNamespaces loads all namespaces saved to graph. func (g *graphObject) LoadNamespaces() error { return g.s.sch.LoadNamespaces(g.s.ctx, g.s.qs, &g.s.ns) } // V is a shorthand for Vertex. func (g *graphObject) NewV(call goja.FunctionCall) goja.Value { return g.NewVertex(call) } // Vertex starts a query path at the given vertex/vertices. No ids means "all vertices". // Signature: ([nodeId],[nodeId]...) // // Arguments: // // * `nodeId` (Optional): A string or list of strings representing the starting vertices. // // Returns: Path object func (g *graphObject) NewVertex(call goja.FunctionCall) goja.Value { qv, err := toQuadValues(exportArgs(call.Arguments)) if err != nil { return throwErr(g.s.vm, err) } return g.s.vm.ToValue(&pathObject{ s: g.s, finals: true, path: path.StartMorphism(qv...), }) } // M is a shorthand for Morphism. func (g *graphObject) NewM() *pathObject { return g.NewMorphism() } // Morphism creates a morphism path object. Unqueryable on it's own, defines one end of the path. // Saving these to variables with // // // javascript // var shorterPath = graph.Morphism().out("foo").out("bar") // // is the common use case. See also: path.follow(), path.followR(). func (g *graphObject) NewMorphism() *pathObject { return &pathObject{ s: g.s, path: path.StartMorphism(), } } // Emit adds data programmatically to the JSON result list. Can be any JSON type. // // // javascript // g.emit({name:"bob"}) // push {"name":"bob"} as a result func (g *graphObject) Emit(call goja.FunctionCall) goja.Value { value := call.Argument(0) if !goja.IsNull(value) && !goja.IsUndefined(value) { val := exportArgs([]goja.Value{value})[0] if val != nil { g.s.send(nil, &Result{Val: val}) } } return goja.Null() } // Backwards compatibility func (g *graphObject) CapitalizedUri(s string) quad.IRI { return g.NewIRI(s) } func (g *graphObject) CapitalizedAddNamespace(pref, ns string) { g.AddNamespace(pref, ns) } func (g *graphObject) CapitalizedAddDefaultNamespaces() { g.AddDefaultNamespaces() } func (g *graphObject) CapitalizedLoadNamespaces() error { return g.LoadNamespaces() } func (g *graphObject) CapitalizedEmit(call goja.FunctionCall) goja.Value { return g.Emit(call) } func oneStringType(fnc func(s string) quad.Value) func(vm *goja.Runtime, call goja.FunctionCall) goja.Value { return func(vm *goja.Runtime, call goja.FunctionCall) goja.Value { args := toStrings(exportArgs(call.Arguments)) if len(args) != 1 { return throwErr(vm, errArgCount2{Expected: 1, Got: len(args)}) } return vm.ToValue(fnc(args[0])) } } func twoStringType(fnc func(s1, s2 string) quad.Value) func(vm *goja.Runtime, call goja.FunctionCall) goja.Value { return func(vm *goja.Runtime, call goja.FunctionCall) goja.Value { args := toStrings(exportArgs(call.Arguments)) if len(args) != 2 { return throwErr(vm, errArgCount2{Expected: 2, Got: len(args)}) } return vm.ToValue(fnc(args[0], args[1])) } } func cmpOpType(op iterator.Operator) func(vm *goja.Runtime, call goja.FunctionCall) goja.Value { return func(vm *goja.Runtime, call goja.FunctionCall) goja.Value { args := exportArgs(call.Arguments) if len(args) != 1 { return throwErr(vm, errArgCount2{Expected: 1, Got: len(args)}) } qv, err := toQuadValue(args[0]) if err != nil { return throwErr(vm, err) } return vm.ToValue(valFilter{f: shape.Comparison{Op: op, Val: qv}}) } } func cmpWildcard(vm *goja.Runtime, call goja.FunctionCall) goja.Value { args := exportArgs(call.Arguments) if len(args) != 1 { return throwErr(vm, errArgCount2{Expected: 1, Got: len(args)}) } pattern, ok := args[0].(string) if !ok { return throwErr(vm, fmt.Errorf("wildcard: unsupported type: %T", args[0])) } return vm.ToValue(valFilter{f: shape.Wildcard{Pattern: pattern}}) } func cmpRegexp(vm *goja.Runtime, call goja.FunctionCall) goja.Value { args := exportArgs(call.Arguments) if len(args) != 1 && len(args) != 2 { return throwErr(vm, errArgCount2{Expected: 1, Got: len(args)}) } v, err := toQuadValue(args[0]) if err != nil { return throwErr(vm, err) } allowRefs := false if len(args) > 1 { b, ok := args[1].(bool) if !ok { return throwErr(vm, fmt.Errorf("expected bool as second argument")) } allowRefs = b } switch vt := v.(type) { case quad.String: if allowRefs { v = quad.IRI(string(vt)) } case quad.IRI: if !allowRefs { return throwErr(vm, errRegexpOnIRI) } case quad.BNode: if !allowRefs { return throwErr(vm, errRegexpOnIRI) } default: return throwErr(vm, fmt.Errorf("regexp: unsupported type: %T", v)) } var ( s string refs bool ) switch v := v.(type) { case quad.String: s = string(v) case quad.IRI: s, refs = string(v), true case quad.BNode: s, refs = string(v), true default: return throwErr(vm, fmt.Errorf("regexp from non-string value: %T", v)) } re, err := regexp.Compile(string(s)) if err != nil { return throwErr(vm, err) } return vm.ToValue(valFilter{f: shape.Regexp{Re: re, Refs: refs}}) } type valFilter struct { f shape.ValueFilter } var defaultEnv = map[string]func(vm *goja.Runtime, call goja.FunctionCall) goja.Value{ "iri": oneStringType(func(s string) quad.Value { return quad.IRI(s) }), "bnode": oneStringType(func(s string) quad.Value { return quad.BNode(s) }), "raw": oneStringType(func(s string) quad.Value { return quad.Raw(s) }), "str": oneStringType(func(s string) quad.Value { return quad.String(s) }), "lang": twoStringType(func(s, lang string) quad.Value { return quad.LangString{Value: quad.String(s), Lang: lang} }), "typed": twoStringType(func(s, typ string) quad.Value { return quad.TypedString{Value: quad.String(s), Type: quad.IRI(typ)} }), "lt": cmpOpType(iterator.CompareLT), "lte": cmpOpType(iterator.CompareLTE), "gt": cmpOpType(iterator.CompareGT), "gte": cmpOpType(iterator.CompareGTE), "regex": cmpRegexp, "like": cmpWildcard, } func unwrap(o interface{}) interface{} { switch v := o.(type) { case *pathObject: o = v.path case []interface{}: for i, val := range v { v[i] = unwrap(val) } case map[string]interface{}: for k, val := range v { v[k] = unwrap(val) } } return o } func exportArgs(args []goja.Value) []interface{} { if len(args) == 0 { return nil } out := make([]interface{}, 0, len(args)) for _, a := range args { o := a.Export() out = append(out, unwrap(o)) } return out } func toInt(o interface{}) (int, bool) { switch v := o.(type) { case int: return v, true case int64: return int(v), true case float64: return int(v), true default: return 0, false } } func toQuadValue(o interface{}) (quad.Value, error) { var qv quad.Value switch v := o.(type) { case quad.Value: qv = v case string: qv = quad.StringToValue(v) case bool: qv = quad.Bool(v) case int: qv = quad.Int(v) case int64: qv = quad.Int(v) case float64: if float64(int(v)) == v { qv = quad.Int(int64(v)) } else { qv = quad.Float(v) } case time.Time: qv = quad.Time(v) default: return nil, errNotQuadValue{Val: o} } return qv, nil } func toQuadValues(objs []interface{}) ([]quad.Value, error) { if len(objs) == 0 { return nil, nil } vals := make([]quad.Value, 0, len(objs)) for _, o := range objs { qv, err := toQuadValue(o) if err != nil { return nil, err } vals = append(vals, qv) } return vals, nil } func toStrings(objs []interface{}) []string { if len(objs) == 0 { return nil } var out = make([]string, 0, len(objs)) for _, o := range objs { switch v := o.(type) { case string: out = append(out, v) case quad.Value: out = append(out, quad.StringOf(v)) case []string: out = append(out, v...) case []interface{}: out = append(out, toStrings(v)...) default: panic(fmt.Errorf("expected string, got: %T", o)) } } return out } func toVia(via []interface{}) []interface{} { if len(via) == 0 { return nil } else if len(via) == 1 { if via[0] == nil { return nil } else if v, ok := via[0].([]interface{}); ok { return toVia(v) } else if v, ok := via[0].([]string); ok { arr := make([]interface{}, 0, len(v)) for _, s := range v { arr = append(arr, s) } return toVia(arr) } } for i := range via { if _, ok := via[i].(*path.Path); ok { // bypass } else if vp, ok := via[i].(*pathObject); ok { via[i] = vp.path } else if qv, err := toQuadValue(via[i]); err == nil { via[i] = qv } else { panic(fmt.Errorf("unsupported type: %T", via[i])) } } return via } func toViaData(objs []interface{}) (predicates []interface{}, tags []string, ok bool) { if len(objs) != 0 { predicates = toVia([]interface{}{objs[0]}) } if len(objs) > 1 { tags = toStrings(objs[1:]) } ok = true return } func toViaDepthData(objs []interface{}) (predicates []interface{}, maxDepth int, tags []string, ok bool) { if len(objs) != 0 { predicates = toVia([]interface{}{objs[0]}) } if len(objs) > 1 { maxDepth, ok = toInt(objs[1]) if ok { if len(objs) > 2 { tags = toStrings(objs[2:]) } } else { tags = toStrings(objs[1:]) } } ok = true return } func throwErr(vm *goja.Runtime, err error) goja.Value { panic(vm.ToValue(err)) } ================================================ FILE: query/gizmo/errors.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gizmo import "fmt" var ( errNoVia = fmt.Errorf("expected predicate list") errRegexpOnIRI = fmt.Errorf("regexps are not allowed on IRIs") ) type errArgCount2 struct { Expected int Got int } func (e errArgCount2) Error() string { return fmt.Sprintf("expected %d argument, got %d", e.Expected, e.Got) } type errArgCount struct { Got int } func (e errArgCount) Error() string { return fmt.Sprintf("unexpected arguments count: %d", e.Got) } type errNotQuadValue struct { Val interface{} } func (e errNotQuadValue) Error() string { return fmt.Sprintf("not a quad.Value: %T", e.Val) } ================================================ FILE: query/gizmo/finals.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gizmo import ( "github.com/dop251/goja" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/quad" ) const TopResultTag = "id" // GetLimit is the same as All, but limited to the first N unique nodes at the end of the path, and each of their possible traversals. func (p *pathObject) GetLimit(limit int) error { it := p.buildIteratorTree() it = iterator.Tag(it, TopResultTag) p.s.limit = limit p.s.count = 0 return p.s.runIterator(it) } // All executes the query and adds the results, with all tags, as a string-to-string (tag to node) map in the output set, one for each path that a traversal could take. func (p *pathObject) All() error { return p.GetLimit(p.s.limit) } func (p *pathObject) toArray(call goja.FunctionCall, withTags bool) goja.Value { args := exportArgs(call.Arguments) if len(args) > 1 { return throwErr(p.s.vm, errArgCount2{Expected: 1, Got: len(args)}) } limit := -1 if len(args) > 0 { limit, _ = toInt(args[0]) } it := p.buildIteratorTree() it = iterator.Tag(it, TopResultTag) var ( array interface{} err error ) if !withTags { array, err = p.s.runIteratorToArrayNoTags(it, limit) } else { array, err = p.s.runIteratorToArray(it, limit) } if err != nil { return throwErr(p.s.vm, err) } return p.s.vm.ToValue(array) } // ToArray executes a query and returns the results at the end of the query path as an JS array. // // Example: // // javascript // // bobFollowers contains an Array of followers of bob (alice, charlie, dani). // var bobFollowers = g.V("").In("").ToArray() func (p *pathObject) ToArray(call goja.FunctionCall) goja.Value { return p.toArray(call, false) } // TagArray is the same as ToArray, but instead of a list of top-level nodes, returns an Array of tag-to-string dictionaries, much as All would, except inside the JS environment. // // Example: // // javascript // // bobTags contains an Array of followers of bob (alice, charlie, dani). // var bobTags = g.V("").Tag("name").In("").TagArray() // // nameValue should be the string "" // var nameValue = bobTags[0]["name"] func (p *pathObject) TagArray(call goja.FunctionCall) goja.Value { return p.toArray(call, true) } func (p *pathObject) toValue(withTags bool) (interface{}, error) { it := p.buildIteratorTree() it = iterator.Tag(it, TopResultTag) const limit = 1 if !withTags { array, err := p.s.runIteratorToArrayNoTags(it, limit) if err != nil { return nil, err } if len(array) == 0 { return nil, nil } return array[0], nil } array, err := p.s.runIteratorToArray(it, limit) if err != nil { return nil, err } if len(array) == 0 { return nil, nil } return array[0], nil } // ToValue is the same as ToArray, but limited to one result node. func (p *pathObject) ToValue() (interface{}, error) { return p.toValue(false) } // TagValue is the same as TagArray, but limited to one result node. Returns a tag-to-string map. func (p *pathObject) TagValue() (interface{}, error) { return p.toValue(true) } // Map is a alias for ForEach. func (p *pathObject) Map(call goja.FunctionCall) goja.Value { return p.ForEach(call) } // ForEach calls callback(data) for each result, where data is the tag-to-string map as in All case. // Signature: (callback) or (limit, callback) // // Arguments: // // * `limit` (Optional): An integer value on the first `limit` paths to process. // * `callback`: A javascript function of the form `function(data)` // // Example: // // javascript // // Simulate query.All().All() // graph.V("").ForEach(function(d) { g.Emit(d) } ) func (p *pathObject) ForEach(call goja.FunctionCall) goja.Value { it := p.buildIteratorTree() it = iterator.Tag(it, TopResultTag) if n := len(call.Arguments); n != 1 && n != 2 { return throwErr(p.s.vm, errArgCount{Got: len(call.Arguments)}) } callback := call.Argument(len(call.Arguments) - 1) args := exportArgs(call.Arguments[:len(call.Arguments)-1]) limit := -1 if len(args) != 0 { limit, _ = toInt(args[0]) } err := p.s.runIteratorWithCallback(it, callback, call, limit) if err != nil { return throwErr(p.s.vm, err) } return goja.Null() } // Count returns a number of results and returns it as a value. // // Example: // // javascript // // Save count as a variable // var n = g.V().count() // // Send it as a query result // g.emit(n) func (p *pathObject) Count() (int64, error) { it := p.buildIteratorTree() return p.s.countResults(it) } // Backwards compatibility func (p *pathObject) CapitalizedGetLimit(limit int) error { return p.GetLimit(limit) } func (p *pathObject) CapitalizedAll() error { return p.All() } func (p *pathObject) CapitalizedtoArray(call goja.FunctionCall, withTags bool) goja.Value { return p.toArray(call, withTags) } func (p *pathObject) CapitalizedToArray(call goja.FunctionCall) goja.Value { return p.ToArray(call) } func (p *pathObject) CapitalizedTagArray(call goja.FunctionCall) goja.Value { return p.TagArray(call) } func (p *pathObject) CapitalizedtoValue(withTags bool) (interface{}, error) { return p.toValue(withTags) } func (p *pathObject) CapitalizedToValue() (interface{}, error) { return p.ToValue() } func (p *pathObject) CapitalizedTagValue() (interface{}, error) { return p.TagValue() } func (p *pathObject) CapitalizedMap(call goja.FunctionCall) goja.Value { return p.Map(call) } func (p *pathObject) CapitalizedForEach(call goja.FunctionCall) goja.Value { return p.ForEach(call) } func (p *pathObject) CapitalizedCount() (int64, error) { return p.Count() } func quadValueToString(v quad.Value) string { if s, ok := v.(quad.String); ok { return string(s) } return quad.StringOf(v) } ================================================ FILE: query/gizmo/gizmo.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gizmo import ( "context" "errors" "fmt" "reflect" "sort" "strings" "unicode" "unicode/utf8" "github.com/dop251/goja" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/jsonld" "github.com/cayleygraph/quad/voc" ) const Name = "gizmo" func init() { query.RegisterLanguage(query.Language{ Name: Name, Session: func(qs graph.QuadStore) query.Session { return NewSession(qs) }, }) } func NewSession(qs graph.QuadStore) *Session { s := &Session{ ctx: context.Background(), sch: schema.NewConfig(), qs: qs, limit: -1, } if err := s.buildEnv(); err != nil { panic(err) } return s } func lcFirst(str string) string { rune, size := utf8.DecodeRuneInString(str) return string(unicode.ToLower(rune)) + str[size:] } type fieldNameMapper struct{} func (fieldNameMapper) FieldName(t reflect.Type, f reflect.StructField) string { return lcFirst(f.Name) } const constructMethodPrefix = "New" const backwardsCompatibilityPrefix = "Capitalized" func (fieldNameMapper) MethodName(t reflect.Type, m reflect.Method) string { if strings.HasPrefix(m.Name, backwardsCompatibilityPrefix) { return strings.TrimPrefix(m.Name, backwardsCompatibilityPrefix) } if strings.HasPrefix(m.Name, constructMethodPrefix) { return strings.TrimPrefix(m.Name, constructMethodPrefix) } return lcFirst(m.Name) } type Session struct { qs graph.QuadStore vm *goja.Runtime ns voc.Namespaces sch *schema.Config col query.Collation last string p *goja.Program out chan *Result ctx context.Context limit int count int err error } func (s *Session) context() context.Context { return s.ctx } func (s *Session) buildEnv() error { if s.vm != nil { return nil } s.vm = goja.New() s.vm.SetFieldNameMapper(fieldNameMapper{}) s.vm.Set("graph", &graphObject{s: s}) s.vm.Set("g", s.vm.Get("graph")) for name, val := range defaultEnv { fnc := val s.vm.Set(name, func(call goja.FunctionCall) goja.Value { return fnc(s.vm, call) }) } return nil } func (s *Session) quadValueToNative(v quad.Value) interface{} { if v == nil { return nil } if s.col == query.JSONLD { return jsonld.FromValue(v) } out := v.Native() if nv, ok := out.(quad.Value); ok && v == nv { return quad.StringOf(v) } return out } func (s *Session) tagsToValueMap(m map[string]graph.Ref) (map[string]interface{}, error) { outputMap := make(map[string]interface{}) for k, v := range m { nv, err := s.qs.NameOf(v) if err != nil { return nil, err } if o := s.quadValueToNative(nv); o != nil { outputMap[k] = o } } if len(outputMap) == 0 { return nil, nil } return outputMap, nil } func (s *Session) runIteratorToArray(it iterator.Shape, limit int) ([]map[string]interface{}, error) { ctx := s.context() output := make([]map[string]interface{}, 0) err := iterator.Iterate(ctx, it).Limit(limit).TagEach(func(tags map[string]graph.Ref) error { tm, err := s.tagsToValueMap(tags) if err != nil { return err } if tm != nil { output = append(output, tm) } return nil }) if err != nil { return nil, err } return output, nil } func (s *Session) runIteratorToArrayNoTags(it iterator.Shape, limit int) ([]interface{}, error) { ctx := s.context() output := make([]interface{}, 0) err := iterator.Iterate(ctx, it).Paths(false).Limit(limit).EachValue(s.qs, func(v quad.Value) error { if o := s.quadValueToNative(v); o != nil { output = append(output, o) } return nil }) if err != nil { return nil, err } return output, nil } func (s *Session) runIteratorWithCallback(it iterator.Shape, callback goja.Value, this goja.FunctionCall, limit int) error { fnc, ok := goja.AssertFunction(callback) if !ok { return fmt.Errorf("expected js callback function") } ctx, cancel := context.WithCancel(s.context()) defer cancel() return iterator.Iterate(ctx, it).Paths(true).Limit(limit).TagEach(func(tags map[string]graph.Ref) error { tm, err := s.tagsToValueMap(tags) if err != nil || tm == nil { return err } _, err = fnc(this.This, s.vm.ToValue(tm)) if err != nil { cancel() } return err }) } func (s *Session) send(ctx context.Context, r *Result) bool { if s.limit > 0 && s.count >= s.limit { return false } if s.out == nil { return false } if ctx == nil { ctx = s.ctx } select { case s.out <- r: case <-ctx.Done(): return false } s.count++ return s.limit <= 0 || s.count < s.limit } func (s *Session) runIterator(it iterator.Shape) error { ctx, cancel := context.WithCancel(s.context()) defer cancel() stop := false err := iterator.Iterate(ctx, it).Paths(true).TagEach(func(tags map[string]graph.Ref) error { if !s.send(ctx, &Result{Tags: tags}) { cancel() stop = true } return nil }) if stop { err = nil } return err } func (s *Session) countResults(it iterator.Shape) (int64, error) { return iterator.Iterate(s.context(), it).Paths(true).Count() } type Result struct { Meta bool Val interface{} Tags map[string]graph.Ref } func (r *Result) Result() interface{} { if r.Tags != nil { return r.Tags } return r.Val } func (s *Session) compile(qu string) error { var p *goja.Program if s.last == qu && s.last != "" { p = s.p } else { var err error p, err = goja.Compile("", qu, false) if err != nil { return err } s.last, s.p = qu, p } return nil } func (s *Session) run() (goja.Value, error) { v, err := s.vm.RunProgram(s.p) if e, ok := err.(*goja.Exception); ok && e.Value() != nil { if er, ok := e.Value().Export().(error); ok { err = er } } return v, err } func (s *Session) Execute(ctx context.Context, qu string, opt query.Options) (query.Iterator, error) { switch opt.Collation { case query.Raw, query.JSON, query.JSONLD, query.REPL: default: return nil, &query.ErrUnsupportedCollation{Collation: opt.Collation} } if err := s.compile(qu); err != nil { return nil, err } s.limit = opt.Limit s.count = 0 ctx, cancel := context.WithCancel(context.Background()) s.ctx = ctx s.col = opt.Collation return &results{ col: opt.Collation, s: s, ctx: ctx, cancel: cancel, }, nil } type results struct { s *Session col query.Collation ctx context.Context cancel func() running bool errc chan error err error cur *Result } func (it *results) stop(err error) { it.cancel() if !it.running { return } it.s.vm.Interrupt(err) it.running = false } func (it *results) Next(ctx context.Context) bool { if it.errc == nil { it.s.out = make(chan *Result) it.errc = make(chan error, 1) it.running = true go func() { defer close(it.errc) v, err := it.s.run() if err != nil { it.errc <- err return } if !goja.IsNull(v) && !goja.IsUndefined(v) { it.s.send(it.ctx, &Result{Meta: true, Val: v.Export()}) } }() } select { case r := <-it.s.out: it.cur = r return true case err := <-it.errc: if err != nil { it.err = err } return false case <-ctx.Done(): it.err = ctx.Err() it.stop(it.err) return false } } func (it *results) Result() interface{} { if it.cur == nil { return nil } switch it.col { case query.Raw: return it.cur case query.JSON, query.JSONLD: return it.jsonResult() case query.REPL: return it.replResult() } return nil } func (it *results) jsonResult() interface{} { data := it.cur if data.Meta { return nil } if data.Val != nil { return data.Val } obj := make(map[string]interface{}) tags := data.Tags var tagKeys []string for k := range tags { tagKeys = append(tagKeys, k) } sort.Strings(tagKeys) for _, k := range tagKeys { name, err := it.s.qs.NameOf(tags[k]) if err != nil { it.err = err return nil } if name != nil { obj[k] = it.s.quadValueToNative(name) } else { delete(obj, k) } } return obj } func (it *results) replResult() interface{} { data := it.cur if data.Meta { if data.Val != nil { s := data.Val switch s.(type) { case *pathObject, *graphObject: s = "[internal Iterator]" } return fmt.Sprintln("=>", s) } return fmt.Sprintln("=>", nil) } var out string out = fmt.Sprintln("****") if data.Val == nil { tags := data.Tags tagKeys := make([]string, len(tags)) i := 0 for k := range tags { tagKeys[i] = k i++ } sort.Strings(tagKeys) for _, k := range tagKeys { if k == "$_" { continue } knv, err := it.s.qs.NameOf(tags[k]) if err != nil { // ignore continue } out += fmt.Sprintf("%s : %s\n", k, quadValueToString(knv)) } } else { switch export := data.Val.(type) { case map[string]string: for k, v := range export { out += fmt.Sprintf("%s : %s\n", k, v) } case map[string]interface{}: for k, v := range export { out += fmt.Sprintf("%s : %v\n", k, v) } default: out += fmt.Sprintf("%s\n", data.Val) } } return out } func (it *results) Err() error { return it.err } func (it *results) Close() error { it.stop(errors.New("iterator closed")) return nil } ================================================ FILE: query/gizmo/gizmo_test.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gizmo import ( "context" "fmt" "reflect" "sort" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest/testutil" _ "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/query" _ "github.com/cayleygraph/cayley/writer" "github.com/cayleygraph/quad" // register global namespace for tests _ "github.com/cayleygraph/quad/voc/rdf" ) // This is a simple test graph used for testing // // +-------+ +------+ // | alice |----- ->| fred |<-- // +-------+ \---->+-------+-/ +------+ \-+-------+ // ----->| #bob# | | | emily | // +---------+--/ --->+-------+ | +-------+ // | charlie | / v // +---------+ / +--------+ // \--- +--------+ | #greg# | // \-->| #dani# |------------>+--------+ // +--------+ // func makeTestSession(data []quad.Quad) *Session { qs, _ := graph.NewQuadStore("memstore", "", nil) w, _ := graph.NewQuadWriter("single", qs, nil) for _, t := range data { w.AddQuad(t) } return NewSession(qs) } func intVal(v int) string { return quad.Int(v).String() } const multiGraphTestFile = "../../data/testdata_multigraph.nq" var testQueries = []struct { message string data []quad.Quad query string limit int tag string file string expect []string err bool // TODO(dennwc): define error types for Gizmo and handle them }{ // Simple query tests. { message: "get a single vertex", query: ` g.V("").all() `, expect: []string{""}, }, { message: "get a single vertex (legacy)", query: ` g.V("").All() `, expect: []string{""}, }, { message: "get a single vertex (legacy)", query: ` g.V("").All() `, expect: []string{""}, }, { message: "use .getLimit", query: ` g.V().getLimit(5) `, expect: []string{"", "", "", "", ""}, }, { message: "get a single vertex (IRI)", query: ` g.V(iri("alice")).all() `, expect: []string{""}, }, { message: "use .out()", query: ` g.V("").out("").all() `, expect: []string{""}, }, { message: "use .out() (IRI)", query: ` g.V(iri("alice")).out(iri("follows")).all() `, expect: []string{""}, }, { message: "use .out() (any)", query: ` g.V("").out().all() `, expect: []string{"", "cool_person"}, }, { message: "use .in()", query: ` g.V("").in("").all() `, expect: []string{"", "", ""}, }, { message: "use .in() (any)", query: ` g.V("").in().all() `, expect: []string{"", "", ""}, }, { message: "use .in() with .filter()", query: ` g.V("").in("").filter(gt(iri("c")),lt(iri("d"))).all() `, expect: []string{""}, }, { message: "use .in() with .filter(regex)", query: ` g.V("").in("").filter(regex("ar?li.*e")).all() `, expect: nil, }, { message: "use .in() with .filter(prefix)", query: ` g.V("").in("").filter(like("al%")).all() `, expect: []string{""}, }, { message: "use .in() with .filter(wildcard)", query: ` g.V("").in("").filter(like("a?i%e")).all() `, expect: []string{""}, }, { message: "use .in() with .filter(regex with IRIs)", query: ` g.V("").in("").filter(regex("ar?li.*e", true)).all() `, expect: []string{"", ""}, }, { message: "use .in() with .filter(regex with IRIs)", query: ` g.V("").in("").filter(regex(iri("ar?li.*e"))).all() `, err: true, }, { message: "use .in() with .filter(regex,gt)", query: ` g.V("").in("").filter(regex("ar?li.*e", true),gt(iri("c"))).all() `, expect: []string{""}, }, { message: "filter with a wrong type", query: ` g.V().filter(//).all() `, err: true, }, { message: "use .both()", query: ` g.V("").both("").all() `, expect: []string{"", "", ""}, }, { message: "use .both() with tag", query: ` g.V("").both(null, "pred").all() `, tag: "pred", expect: []string{"", "", ""}, }, { message: "use .tag()-.is()-.back()", query: ` g.V("").in("").tag("foo").out("").is("cool_person").back("foo").all() `, expect: []string{""}, }, { message: "separate .tag()-.is()-.back()", query: ` x = g.V("").out("").tag("foo").out("").is("cool_person").back("foo") x.in("").is("").back("foo").all() `, expect: []string{""}, }, { message: "do multiple .back()", query: ` g.V("").out("").as("f").out("").out("").is("cool_person").back("f").in("").in("").as("acd").out("").is("cool_person").back("f").all() `, tag: "acd", expect: []string{""}, }, { message: "use Except to filter out a single vertex", query: ` g.V("", "").except(g.V("")).all() `, expect: []string{""}, }, { message: "use chained Except", query: ` g.V("", "", "").except(g.V("")).except(g.V("")).all() `, expect: []string{""}, }, { message: "use Unique", query: ` g.V("", "", "").out("").unique().all() `, expect: []string{"", "", ""}, }, // Morphism tests. { message: "show simple morphism", query: ` grandfollows = g.M().out("").out("") g.V("").follow(grandfollows).all() `, expect: []string{"", "", ""}, }, { message: "show reverse morphism", query: ` grandfollows = g.M().out("").out("") g.V("").followR(grandfollows).all() `, expect: []string{"", "", ""}, }, // Intersection tests. { message: "show simple intersection", query: ` function follows(x) { return g.V(x).out("") } follows("").and(follows("")).all() `, expect: []string{""}, }, { message: "show simple morphism intersection", query: ` grandfollows = g.M().out("").out("") function gfollows(x) { return g.V(x).follow(grandfollows) } gfollows("").and(gfollows("")).all() `, expect: []string{""}, }, { message: "show double morphism intersection", query: ` grandfollows = g.M().out("").out("") function gfollows(x) { return g.V(x).follow(grandfollows) } gfollows("").and(gfollows("")).and(gfollows("")).all() `, expect: []string{""}, }, { message: "show reverse intersection", query: ` grandfollows = g.M().out("").out("") g.V("").followR(grandfollows).intersect(g.V("").followR(grandfollows)).all() `, expect: []string{""}, }, { message: "show standard sort of morphism intersection, continue follow", query: `gfollowers = g.M().in("").in("") function cool(x) { return g.V(x).as("a").out("").is("cool_person").back("a") } cool("").follow(gfollowers).intersect(cool("").follow(gfollowers)).all() `, expect: []string{""}, }, { message: "test Or()", query: ` g.V("").out("").or(g.V().has("", "cool_person")).all() `, expect: []string{"", "", "", ""}, }, // Has tests. { message: "show a simple Has", query: ` g.V().has("", "cool_person").all() `, expect: []string{"", "", ""}, }, { message: "show a simple HasR", query: ` g.V().hasR("", "").all() `, expect: []string{"cool_person"}, }, { message: "show a double Has", query: ` g.V().has("", "cool_person").has("", "").all() `, expect: []string{""}, }, { message: "show a Has with filter", query: ` g.V().has("", gt("")).all() `, expect: []string{"", "", "", ""}, }, // Skip/Limit tests. { message: "use Limit", query: ` g.V().has("", "cool_person").limit(2).all() `, expect: []string{"", ""}, }, { message: "use Skip", query: ` g.V().has("", "cool_person").skip(2).all() `, expect: []string{""}, }, { message: "use Skip and Limit", query: ` g.V().has("", "cool_person").skip(1).limit(1).all() `, expect: []string{""}, }, { message: "show Count", query: ` g.V().has("").count() `, expect: []string{"5"}, }, { message: "use Count value", query: ` g.emit(g.V().has("").count()+1) `, expect: []string{"6"}, }, // Tag tests. { message: "show a simple save", query: ` g.V().save("", "somecool").all() `, tag: "somecool", expect: []string{"cool_person", "cool_person", "cool_person", "smart_person", "smart_person"}, }, { message: "show a simple save optional", query: ` g.V("","").out("").saveOpt("", "somecool").all() `, tag: "somecool", expect: []string{"cool_person", "cool_person"}, }, { message: "save iri no tag", query: ` g.V().save(g.IRI("status")).all() `, tag: "", expect: []string{"cool_person", "cool_person", "cool_person", "smart_person", "smart_person"}, }, { message: "show a simple saveR", query: ` g.V("cool_person").saveR("", "who").all() `, tag: "who", expect: []string{"", "", ""}, }, { message: "show an out save", query: ` g.V("").out(null, "pred").all() `, tag: "pred", expect: []string{"", "", ""}, }, { message: "show a tag list", query: ` g.V("").out(null, ["pred", "foo", "bar"]).all() `, tag: "foo", expect: []string{"", "", ""}, }, { message: "show a pred list", query: ` g.V("").out(["", ""]).all() `, expect: []string{"", "", "cool_person"}, }, { message: "show a predicate path", query: ` g.V("").out(g.V(""), "pred").all() `, expect: []string{"", ""}, }, { message: "list all bob's incoming predicates", query: ` g.V("").inPredicates().all() `, expect: []string{""}, }, { message: "save all bob's incoming predicates", query: ` g.V("").saveInPredicates("pred").all() `, expect: []string{"", "", ""}, tag: "pred", }, { message: "list all labels", query: ` g.V().labels().all() `, expect: []string{""}, }, { message: "list all in predicates", query: ` g.V().inPredicates().all() `, expect: []string{"", "", ""}, }, { message: "list all out predicates", query: ` g.V().outPredicates().all() `, expect: []string{"", "", ""}, }, { message: "traverse using LabelContext", query: ` g.V("").labelContext("").out("").all() `, expect: []string{"smart_person"}, }, { message: "open and close a LabelContext", query: ` g.V().labelContext("").in("").labelContext(null).in("").all() `, expect: []string{"", ""}, }, { message: "issue #254", query: `g.V({"id":""}).all()`, expect: nil, err: true, }, { message: "roundtrip values", query: ` v = g.V("").toValue() s = g.V(v).out("").toValue() g.V(s).all() `, expect: []string{"cool_person"}, }, { message: "roundtrip values (tag map)", query: ` v = g.V("").tagValue() s = g.V(v.id).out("").tagValue() g.V(s.id).all() `, expect: []string{"cool_person"}, }, { message: "show ToArray", query: ` arr = g.V("").in("").toArray() for (i in arr) g.emit(arr[i]); `, expect: []string{"", "", ""}, }, { message: "show ToArray with limit", query: ` arr = g.V("").in("").toArray(2) for (i in arr) g.emit(arr[i]); `, expect: []string{"", ""}, }, { message: "show ForEach", query: ` g.V("").in("").forEach(function(o){g.emit(o.id)}); `, expect: []string{"", "", ""}, }, { message: "show ForEach with limit", query: ` g.V("").in("").forEach(2, function(o){g.emit(o.id)}); `, expect: []string{"", ""}, }, { message: "clone paths", query: ` var alice = g.V('') g.emit(alice.toValue()) var out = alice.out('') g.emit(out.toValue()) g.emit(alice.toValue()) `, expect: []string{"", "", ""}, }, { message: "default namespaces", query: ` g.addDefaultNamespaces() g.emit(g.IRI('rdf:type')) `, expect: []string{""}, }, { message: "add namespace", query: ` g.addNamespace('ex','http://example.net/') g.emit(g.IRI('ex:alice')) `, expect: []string{""}, }, { message: "recursive follow", query: ` g.V("").followRecursive("").all(); `, expect: []string{"", "", "", ""}, }, { message: "recursive follow tag", query: ` g.V("").followRecursive("", "depth").all(); `, tag: "depth", expect: []string{intVal(1), intVal(1), intVal(2), intVal(2)}, }, { message: "recursive follow path", query: ` g.V("").followRecursive(g.V().out("")).all(); `, expect: []string{"", "", "", ""}, }, { message: "find non-existent", query: ` g.V('').forEach(function(d){ g.emit(d); }) `, expect: nil, }, { message: "default limit All", query: ` g.V().all() `, limit: issue718Limit, data: issue718Graph(), expect: issue718Nodes(), }, { message: "issue #758. Verify saveOpt respects label context", query: ` g.V("").labelContext("").saveOpt("", "statusTag").all() `, tag: "statusTag", file: multiGraphTestFile, expect: []string{"smart_person"}, }, { message: "issue #758. Verify saveR respects label context.", query: ` g.V("smart_person").labelContext("").saveR("", "who").all() `, tag: "who", file: multiGraphTestFile, expect: []string{""}, }, { message: "use order", query: ` g.V().order().all() `, expect: []string{ "", "", "", "", "", "", "", "", "", "", "", "", "cool_person", "smart_person", }, }, { message: "use order tags", query: ` g.V().Tag("target").order().all() `, tag: "target", expect: []string{ "", "", "", "", "", "", "", "", "", "", "", "", "cool_person", "smart_person", }, }, } func runQueryGetTag(rec func(), g []quad.Quad, qu string, tag string, limit int) ([]string, error) { js := makeTestSession(g) ctx := context.TODO() it, err := js.Execute(ctx, qu, query.Options{ Collation: query.Raw, Limit: limit, }) if err != nil { return nil, err } defer it.Close() defer rec() var results []string for it.Next(ctx) { data := it.Result().(*Result) if data.Val == nil { if val := data.Tags[tag]; val != nil { nv, err := js.qs.NameOf(val) if err != nil { return nil, err } results = append(results, quadValueToString(nv)) } } else { switch v := data.Val.(type) { case string: results = append(results, v) default: results = append(results, fmt.Sprint(v)) } } } if err := it.Err(); err != nil { return results, err } return results, nil } func TestGizmo(t *testing.T) { simpleGraph := testutil.LoadGraph(t, "../../data/testdata.nq") multiGraph := testutil.LoadGraph(t, multiGraphTestFile) for _, test := range testQueries { test := test t.Run(test.message, func(t *testing.T) { rec := func() { if r := recover(); r != nil { t.Errorf("Unexpected panic on %s: %v", test.message, r) } } defer rec() if test.tag == "" { test.tag = TopResultTag } quads := simpleGraph if test.file == multiGraphTestFile { quads = multiGraph } if test.data != nil { quads = test.data } limit := test.limit if limit == 0 { limit = -1 } got, err := runQueryGetTag(rec, quads, test.query, test.tag, limit) if err != nil { if test.err { return //expected } t.Error(err) } sort.Strings(got) sort.Strings(test.expect) if !reflect.DeepEqual(got, test.expect) { t.Errorf("got: %v expected: %v", got, test.expect) } }) } } var issue160TestGraph = []quad.Quad{ quad.MakeRaw("alice", "follows", "bob", ""), quad.MakeRaw("bob", "follows", "alice", ""), quad.MakeRaw("charlie", "follows", "bob", ""), quad.MakeRaw("dani", "follows", "charlie", ""), quad.MakeRaw("dani", "follows", "alice", ""), quad.MakeRaw("alice", "is", "cool", ""), quad.MakeRaw("bob", "is", "not cool", ""), quad.MakeRaw("charlie", "is", "cool", ""), quad.MakeRaw("danie", "is", "not cool", ""), } func TestIssue160(t *testing.T) { qu := `g.V().tag('query').out(raw('follows')).out(raw('follows')).forEach(function (item) { if (item.id !== item.query) g.emit({ id: item.id }); })` expect := []string{ "****\nid : alice\n", "****\nid : bob\n", "****\nid : bob\n", } ses := makeTestSession(issue160TestGraph) ctx := context.TODO() it, err := ses.Execute(ctx, qu, query.Options{ Collation: query.REPL, Limit: 100, }) if err != nil { t.Fatal(err) } defer it.Close() var got []string for it.Next(ctx) { func() { defer func() { if r := recover(); r != nil { t.Errorf("Unexpected panic: %v", r) } }() got = append(got, it.Result().(string)) }() } sort.Strings(got) if !reflect.DeepEqual(got, expect) { t.Errorf("Unexpected result, got: %q expected: %q", got, expect) } } const issue718Limit = 5 func issue718Graph() []quad.Quad { var quads []quad.Quad for i := 0; i < issue718Limit; i++ { n := fmt.Sprintf("n%d", i+1) quads = append(quads, quad.MakeIRI("a", "b", n, "")) } return quads } func issue718Nodes() []string { var nodes []string nodes = append(nodes, "", "") for i := 0; i < issue718Limit-2; i++ { n := fmt.Sprintf("", i+1) nodes = append(nodes, n) } return nodes } ================================================ FILE: query/gizmo/traversals.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gizmo // Adds special traversal functions to JS Gizmo objects. Most of these just build the chain of objects, and won't often need the session. import ( "errors" "fmt" "github.com/dop251/goja" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) // pathObject is a Path object in Gizmo. // // Both `.Morphism()` and `.Vertex()` create path objects, which provide the following traversal methods. // Note that `.Vertex()` returns a query object, which is a subclass of path object. // // For these examples, suppose we have the following graph: // // +-------+ +------+ // | alice |----- ->| fred |<-- // +-------+ \---->+-------+-/ +------+ \-+-------+ // ----->| #bob# | | |*emily*| // +---------+--/ --->+-------+ | +-------+ // | charlie | / v // +---------+ / +--------+ // \--- +--------+ |*#greg#*| // \-->| #dani# |------------>+--------+ // +--------+ // // Where every link is a `` relationship, and the nodes with an extra `#` in the name have an extra `` link. As in, // // -- --> "cool_person" // // Perhaps these are the influencers in our community. So too are extra `*`s in the name -- these are our smart people, // according to the `` label, eg, the quad: // // "smart_person" . type pathObject struct { s *Session finals bool path *path.Path } func (p *pathObject) new(np *path.Path) *pathObject { return &pathObject{ s: p.s, finals: p.finals, path: np, } } func (p *pathObject) newVal(np *path.Path) goja.Value { return p.s.vm.ToValue(p.new(np)) } func (p *pathObject) clonePath() *path.Path { np := p.path.Clone() // most likely path will be continued, so we'll put non-capped stack slice // into new path object instead of preserving it in an old one p.path, np = np, p.path return np } func (p *pathObject) buildIteratorTree() iterator.Shape { if p.path == nil { return iterator.NewNull() } return p.path.BuildIteratorOn(p.s.ctx, p.s.qs) } // Filter all paths to ones which, at this point, are on the given node. // Signature: (node, [node..]) // // Arguments: // // * `node`: A string for a node. Can be repeated or a list of strings. // // Example: // // javascript // // Starting from all nodes in the graph, find the paths that follow bob. // // Results in three paths for bob (from alice, charlie and dani).all() // g.V().out("").is("").all() func (p *pathObject) Is(call goja.FunctionCall) goja.Value { args, err := toQuadValues(exportArgs(call.Arguments)) if err != nil { return throwErr(p.s.vm, err) } np := p.clonePath().Is(args...) return p.newVal(np) } func (p *pathObject) inout(call goja.FunctionCall, in bool) goja.Value { preds, tags, ok := toViaData(exportArgs(call.Arguments)) if !ok { return throwErr(p.s.vm, errNoVia) } np := p.clonePath() if in { np = np.InWithTags(tags, preds...) } else { np = np.OutWithTags(tags, preds...) } return p.newVal(np) } // In is inverse of Out. // Starting with the nodes in `path` on the object, follow the quads with predicates defined by `predicatePath` to their subjects. // Signature: ([predicatePath], [tags]) // // Arguments: // // * `predicatePath` (Optional): One of: // * null or undefined: All predicates pointing into this node // * a string: The predicate name to follow into this node // * a list of strings: The predicates to follow into this node // * a query path object: The target of which is a set of predicates to follow. // * `tags` (Optional): One of: // * null or undefined: No tags // * a string: A single tag to add the predicate used to the output set. // * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. // // Example: // // // javascript // // Find the cool people, bob, dani and greg // g.V("cool_person").in("").all() // // Find who follows bob, in this case, alice, charlie, and dani // g.V("").in("").all() // // Find who follows the people emily follows, namely, bob and emily // g.V("").out("").in("").all() func (p *pathObject) In(call goja.FunctionCall) goja.Value { return p.inout(call, true) } // Out is the work-a-day way to get between nodes, in the forward direction. // Starting with the nodes in `path` on the subject, follow the quads with predicates defined by `predicatePath` to their objects. // Signature: ([predicatePath], [tags]) // // Arguments: // // * `predicatePath` (Optional): One of: // * null or undefined: All predicates pointing out from this node // * a string: The predicate name to follow out from this node // * a list of strings: The predicates to follow out from this node // * a query path object: The target of which is a set of predicates to follow. // * `tags` (Optional): One of: // * null or undefined: No tags // * a string: A single tag to add the predicate used to the output set. // * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. // // Example: // // // javascript // // The working set of this is bob and dani // g.V("").out("").all() // // The working set of this is fred, as alice follows bob and bob follows fred. // g.V("").out("").out("").all() // // Finds all things dani points at. Result is bob, greg and cool_person // g.V("").out().all() // // Finds all things dani points at on the status linkage. // // Result is bob, greg and cool_person // g.V("").out(["", ""]).all() // // Finds all things dani points at on the status linkage, given from a separate query path. // // Result is {"id": "cool_person", "pred": ""} // g.V("").out(g.V(""), "pred").all() func (p *pathObject) Out(call goja.FunctionCall) goja.Value { return p.inout(call, false) } // Both follow the predicate in either direction. Same as Out or In. // Signature: ([predicatePath], [tags]) // // Example: // // javascript // // Find all followers/followees of fred. Returns bob, emily and greg // g.V("").both("").all() func (p *pathObject) Both(call goja.FunctionCall) goja.Value { preds, tags, ok := toViaData(exportArgs(call.Arguments)) if !ok { return throwErr(p.s.vm, errNoVia) } np := p.clonePath().BothWithTags(tags, preds...) return p.newVal(np) } func (p *pathObject) follow(ep *pathObject, rev bool) *pathObject { if ep == nil { return p } np := p.clonePath() if rev { np = np.FollowReverse(ep.path) } else { np = np.Follow(ep.path) } return p.new(np) } // Follow is the way to use a path prepared with Morphism. Applies the path chain on the morphism object to the current path. // // Starts as if at the g.M() and follows through the morphism path. // // Example: // // javascript: // var friendOfFriend = g.Morphism().Out("").Out("") // // Returns the followed people of who charlie follows -- a simplistic "friend of my friend" // // and whether or not they have a "cool" status. Potential for recommending followers abounds. // // Returns bob and greg // g.V("").follow(friendOfFriend).has("", "cool_person").all() func (p *pathObject) Follow(path *pathObject) *pathObject { return p.follow(path, false) } // FollowR is the same as Follow but follows the chain in the reverse direction. Flips "In" and "Out" where appropriate, // the net result being a virtual predicate followed in the reverse direction. // // Starts at the end of the morphism and follows it backwards (with appropriate flipped directions) to the g.M() location. // // Example: // // javascript: // var friendOfFriend = g.Morphism().Out("").Out("") // // Returns the third-tier of influencers -- people who follow people who follow the cool people. // // Returns charlie (from bob), charlie (from greg), bob and emily // g.V().has("", "cool_person").followR(friendOfFriend).all() func (p *pathObject) FollowR(path *pathObject) *pathObject { return p.follow(path, true) } // FollowRecursive is the same as Follow but follows the chain recursively. // // Starts as if at the g.M() and follows through the morphism path multiple times, returning all nodes encountered. // // Example: // // javascript: // var friend = g.Morphism().out("") // // Returns all people in Charlie's network. // // Returns bob and dani (from charlie), fred (from bob) and greg (from dani). // g.V("").followRecursive(friend).all() func (p *pathObject) FollowRecursive(call goja.FunctionCall) goja.Value { preds, maxDepth, tags, ok := toViaDepthData(exportArgs(call.Arguments)) if !ok || len(preds) == 0 { return throwErr(p.s.vm, errNoVia) } else if len(preds) != 1 { return throwErr(p.s.vm, fmt.Errorf("expected one predicate or path for recursive follow")) } np := p.clonePath() np = np.FollowRecursive(preds[0], maxDepth, tags) return p.newVal(np) } // And is an alias for Intersect. func (p *pathObject) And(path *pathObject) *pathObject { return p.Intersect(path) } // Intersect filters all paths by the result of another query path. // // This is essentially a join where, at the stage of each path, a node is shared. // Example: // // javascript // var cFollows = g.V("").Out("") // var dFollows = g.V("").Out("") // // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob. // cFollows.Intersect(dFollows).All() // // Equivalently, g.V("").Out("").And(g.V("").Out("")).All() func (p *pathObject) Intersect(path *pathObject) *pathObject { if path == nil { return p } np := p.clonePath().And(path.path) return p.new(np) } // Union returns the combined paths of the two queries. // // Notice that it's per-path, not per-node. Once again, if multiple paths reach the same destination, // they might have had different ways of getting there (and different tags). // See also: `path.Tag()` // // Example: // // javascript // var cFollows = g.V("").Out("") // var dFollows = g.V("").Out("") // // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob (from charlie), dani, bob (from dani), and greg. // cFollows.Union(dFollows).All() func (p *pathObject) Union(path *pathObject) *pathObject { if path == nil { return p } np := p.clonePath().Or(path.path) return p.new(np) } // Or is an alias for Union. func (p *pathObject) Or(path *pathObject) *pathObject { return p.Union(path) } // Back returns current path to a set of nodes on a given tag, preserving all constraints. // // If still valid, a path will now consider their vertex to be the same one as the previously tagged one, // with the added constraint that it was valid all the way here. // Useful for traversing back in queries and taking another route for things that have matched so far. // // Arguments: // // * `tag`: A previous tag in the query to jump back to. // // Example: // // javascript // // Start from all nodes, save them into start, follow any status links, // // jump back to the starting node, and find who follows them. Return the result. // // Results are: // // {"id": "", "start": ""}, // // {"id": "", "start": ""}, // // {"id": "", "start": ""}, // // {"id": "", "start": ""}, // // {"id": "", "start": ""}, // // {"id": "", "start": ""}, // // {"id": "", "start": ""}, // // {"id": "", "start": ""} // g.V().tag("start").out("").back("start").in("").all() func (p *pathObject) Back(tag string) *pathObject { np := p.clonePath().Back(tag) return p.new(np) } // Tag saves a list of nodes to a given tag. // // In order to save your work or learn more about how a path got to the end, we have tags. // The simplest thing to do is to add a tag anywhere you'd like to put each node in the result set. // // Arguments: // // * `tag`: A string or list of strings to act as a result key. The value for tag was the vertex the path was on at the time it reached "Tag" // Example: // // javascript // // Start from all nodes, save them into start, follow any status links, and return the result. // // Results are: // // {"id": "cool_person", "start": ""}, // // {"id": "cool_person", "start": ""}, // // {"id": "cool_person", "start": ""}, // // {"id": "smart_person", "start": ""}, // // {"id": "smart_person", "start": ""} // g.V().tag("start").out("").All() func (p *pathObject) Tag(tags ...string) *pathObject { np := p.clonePath().Tag(tags...) return p.new(np) } // As is an alias for Tag. func (p *pathObject) As(tags ...string) *pathObject { return p.Tag(tags...) } // Has filters all paths which are, at this point, on the subject for the given predicate and object, // but do not follow the path, merely filter the possible paths. // // Usually useful for starting with all nodes, or limiting to a subset depending on some predicate/value pair. // // Signature: (predicate, object) // // Arguments: // // * `predicate`: A string for a predicate node. // * `object`: A string for a object node or a set of filters to find it. // // Example: // // javascript // // Start from all nodes that follow bob -- results in alice, charlie and dani // g.V().has("", "").all() // // People charlie follows who then follow fred. Results in bob. // g.V("").Out("").has("", "").all() // // People with friends who have names sorting lower then "f". // g.V().has("", gt("")).all() func (p *pathObject) Has(call goja.FunctionCall) goja.Value { return p.has(call, false) } // HasR is the same as Has, but sets constraint in reverse direction. func (p *pathObject) HasR(call goja.FunctionCall) goja.Value { return p.has(call, true) } func (p *pathObject) has(call goja.FunctionCall, rev bool) goja.Value { args := exportArgs(call.Arguments) if len(args) == 0 { return throwErr(p.s.vm, errArgCount{Got: len(args)}) } via := args[0] args = args[1:] if vp, ok := via.(*pathObject); ok { via = vp.path } else { var err error via, err = toQuadValue(via) if err != nil { return throwErr(p.s.vm, err) } } if len(args) > 0 { var filt []shape.ValueFilter loop: for _, a := range args { switch a := a.(type) { case valFilter: filt = append(filt, a.f) case []valFilter: for _, s := range a { filt = append(filt, s.f) } default: filt = nil // failed to collect all argument as filters - switch back to nodes-only mode break loop } } if len(filt) > 0 { np := p.clonePath() np = np.HasFilter(via, rev, filt...) return p.newVal(np) } } qv, err := toQuadValues(args) if err != nil { return throwErr(p.s.vm, err) } np := p.clonePath() if rev { np = np.HasReverse(via, qv...) } else { np = np.Has(via, qv...) } return p.newVal(np) } func (p *pathObject) save(call goja.FunctionCall, rev, opt bool) goja.Value { args := exportArgs(call.Arguments) if len(args) > 2 || len(args) == 0 { return throwErr(p.s.vm, errArgCount{Got: len(args)}) } var vtag interface{} = "" if len(args) == 2 { vtag = args[1] } tag, ok := vtag.(string) if !ok { return throwErr(p.s.vm, fmt.Errorf("expected string, got: %T", vtag)) } via := args[0] if vp, ok := via.(*pathObject); ok { via = vp.path if tag == "" { return throwErr(p.s.vm, errors.New("must specify a tag name when saving a path")) } } else { qv, err := toQuadValue(via) via = qv if err != nil { return throwErr(p.s.vm, err) } if tag == "" { if p.s.col == query.JSONLD { switch qv := qv.(type) { case quad.IRI: tag = string(qv) case quad.String: tag = string(qv) default: tag = quad.StringOf(qv) } } else { tag = quad.StringOf(qv) } } } np := p.clonePath() if opt { if rev { np = np.SaveOptionalReverse(via, tag) } else { np = np.SaveOptional(via, tag) } } else { if rev { np = np.SaveReverse(via, tag) } else { np = np.Save(via, tag) } } return p.newVal(np) } // Save saves the object of all quads with predicate into tag, without traversal. // Signature: (predicate, tag) // // Arguments: // // * `predicate`: A string for a predicate node. // * `tag`: A string for a tag key to store the object node. // // Example: // // javascript // // Start from dani and bob and save who they follow into "target" // // Returns: // // {"id" : "", "target": "" }, // // {"id" : "", "target": "" }, // // {"id" : "", "target": "" } // g.V("", "").Save("", "target").All() func (p *pathObject) Save(call goja.FunctionCall) goja.Value { return p.save(call, false, false) } // SaveR is the same as Save, but tags values via reverse predicate. func (p *pathObject) SaveR(call goja.FunctionCall) goja.Value { return p.save(call, true, false) } // SaveOpt is the same as Save, but returns empty tags if predicate does not exists. func (p *pathObject) SaveOpt(call goja.FunctionCall) goja.Value { return p.save(call, false, true) } // SaveOptR is the same as SaveOpt, but tags values via reverse predicate. func (p *pathObject) SaveOptR(call goja.FunctionCall) goja.Value { return p.save(call, true, true) } // Except removes all paths which match query from current path. // // In a set-theoretic sense, this is (A - B). While `g.V().Except(path)` to achieve `U - B = !B` is supported, it's often very slow. // Example: // // javascript // var cFollows = g.V("").Out("") // var dFollows = g.V("").Out("") // // People followed by both charlie (bob and dani) and dani (bob and greg) -- returns bob. // cFollows.Except(dFollows).All() // The set (dani) -- what charlie follows that dani does not also follow. // // Equivalently, g.V("").Out("").Except(g.V("").Out("")).All() func (p *pathObject) Except(path *pathObject) *pathObject { if path == nil { return p } np := p.clonePath().Except(path.path) return p.new(np) } // Unique removes duplicate values from the path. func (p *pathObject) Unique() *pathObject { np := p.clonePath().Unique() return p.new(np) } // Difference is an alias for Except. func (p *pathObject) Difference(path *pathObject) *pathObject { return p.Except(path) } // Labels gets the list of inbound and outbound quad labels func (p *pathObject) Labels() *pathObject { np := p.clonePath().Labels() return p.new(np) } // InPredicates gets the list of predicates that are pointing in to a node. // // Example: // // javascript // // bob only has "" predicates pointing inward // // returns "" // g.V("").InPredicates().All() func (p *pathObject) InPredicates() *pathObject { np := p.clonePath().InPredicates() return p.new(np) } // OutPredicates gets the list of predicates that are pointing out from a node. // // Example: // // javascript // // bob has "" and "" edges pointing outwards // // returns "", "" // g.V("").OutPredicates().All() func (p *pathObject) OutPredicates() *pathObject { np := p.clonePath().OutPredicates() return p.new(np) } // SaveInPredicates tags the list of predicates that are pointing in to a node. // // Example: // // javascript // // bob only has "" predicates pointing inward // // returns {"id":"", "pred":""} // g.V("").SaveInPredicates("pred").All() func (p *pathObject) SaveInPredicates(tag string) *pathObject { np := p.clonePath().SavePredicates(true, tag) return p.new(np) } // SaveOutPredicates tags the list of predicates that are pointing out from a node. // // Example: // // javascript // // bob has "" and "" edges pointing outwards // // returns {"id":"", "pred":""} // g.V("").SaveInPredicates("pred").All() func (p *pathObject) SaveOutPredicates(tag string) *pathObject { np := p.clonePath().SavePredicates(false, tag) return p.new(np) } // LabelContext sets (or removes) the subgraph context to consider in the following traversals. // Affects all In(), Out(), and Both() calls that follow it. The default LabelContext is null (all subgraphs). // Signature: ([labelPath], [tags]) // // Arguments: // // * `predicatePath` (Optional): One of: // * null or undefined: In future traversals, consider all edges, regardless of subgraph. // * a string: The name of the subgraph to restrict traversals to. // * a list of strings: A set of subgraphs to restrict traversals to. // * a query path object: The target of which is a set of subgraphs. // * `tags` (Optional): One of: // * null or undefined: No tags // * a string: A single tag to add the last traversed label to the output set. // * a list of strings: Multiple tags to use as keys to save the label used to the output set. // // Example: // // javascript // // Find the status of people Dani follows // g.V("").out("").out("").all() // // Find only the statuses provided by the smart_graph // g.V("").out("").labelContext("").out("").all() // // Find all people followed by people with statuses in the smart_graph. // g.V().labelContext("").in("").labelContext(null).in("").all() func (p *pathObject) LabelContext(call goja.FunctionCall) goja.Value { labels, tags, ok := toViaData(exportArgs(call.Arguments)) if !ok { return throwErr(p.s.vm, errNoVia) } np := p.clonePath().LabelContextWithTags(tags, labels...) return p.newVal(np) } // Filter applies constraints to a set of nodes. Can be used to filter values by range or match strings. func (p *pathObject) Filter(args ...valFilter) (*pathObject, error) { if len(args) == 0 { return nil, errArgCount{Got: len(args)} } filt := make([]shape.ValueFilter, 0, len(args)) for _, f := range args { if f.f == nil { return nil, errors.New("invalid argument type in filter()") } filt = append(filt, f.f) } np := p.clonePath().Filters(filt...) return p.new(np), nil } // Limit limits a number of nodes for current path. // // Arguments: // // * `limit`: A number of nodes to limit results to. // // Example: // // javascript // // Start from all nodes that follow bob, and limit them to 2 nodes -- results in alice and charlie // g.V().has("", "").limit(2).all() func (p *pathObject) Limit(limit int) *pathObject { np := p.clonePath().Limit(int64(limit)) return p.new(np) } // Skip skips a number of nodes for current path. // // Arguments: // // * `offset`: A number of nodes to skip. // // Example: // // javascript // // Start from all nodes that follow bob, and skip 2 nodes -- results in dani // g.V().has("", "").skip(2).all() func (p *pathObject) Skip(offset int) *pathObject { np := p.clonePath().Skip(int64(offset)) return p.new(np) } func (p *pathObject) Order() *pathObject { np := p.clonePath().Order() return p.new(np) } // Backwards compatibility func (p *pathObject) CapitalizedIs(call goja.FunctionCall) goja.Value { return p.Is(call) } func (p *pathObject) CapitalizedIn(call goja.FunctionCall) goja.Value { return p.In(call) } func (p *pathObject) CapitalizedOut(call goja.FunctionCall) goja.Value { return p.Out(call) } func (p *pathObject) CapitalizedBoth(call goja.FunctionCall) goja.Value { return p.Both(call) } func (p *pathObject) CapitalizedFollow(path *pathObject) *pathObject { return p.Follow(path) } func (p *pathObject) CapitalizedFollowR(path *pathObject) *pathObject { return p.FollowR(path) } func (p *pathObject) CapitalizedFollowRecursive(call goja.FunctionCall) goja.Value { return p.FollowRecursive(call) } func (p *pathObject) CapitalizedAnd(path *pathObject) *pathObject { return p.And(path) } func (p *pathObject) CapitalizedIntersect(path *pathObject) *pathObject { return p.Intersect(path) } func (p *pathObject) CapitalizedUnion(path *pathObject) *pathObject { return p.Union(path) } func (p *pathObject) CapitalizedOr(path *pathObject) *pathObject { return p.Or(path) } func (p *pathObject) CapitalizedBack(tag string) *pathObject { return p.Back(tag) } func (p *pathObject) CapitalizedTag(tags ...string) *pathObject { return p.Tag(tags...) } func (p *pathObject) CapitalizedAs(tags ...string) *pathObject { return p.As(tags...) } func (p *pathObject) CapitalizedHas(call goja.FunctionCall) goja.Value { return p.Has(call) } func (p *pathObject) CapitalizedHasR(call goja.FunctionCall) goja.Value { return p.HasR(call) } func (p *pathObject) CapitalizedSave(call goja.FunctionCall) goja.Value { return p.Save(call) } func (p *pathObject) CapitalizedSaveR(call goja.FunctionCall) goja.Value { return p.SaveR(call) } func (p *pathObject) CapitalizedSaveOpt(call goja.FunctionCall) goja.Value { return p.SaveOpt(call) } func (p *pathObject) CapitalizedSaveOptR(call goja.FunctionCall) goja.Value { return p.SaveOptR(call) } func (p *pathObject) CapitalizedExcept(path *pathObject) *pathObject { return p.Except(path) } func (p *pathObject) CapitalizedUnique() *pathObject { return p.Unique() } func (p *pathObject) CapitalizedDifference(path *pathObject) *pathObject { return p.Difference(path) } func (p *pathObject) CapitalizedLabels() *pathObject { return p.Labels() } func (p *pathObject) CapitalizedInPredicates() *pathObject { return p.InPredicates() } func (p *pathObject) CapitalizedOutPredicates() *pathObject { return p.OutPredicates() } func (p *pathObject) CapitalizedSaveInPredicates(tag string) *pathObject { return p.SaveInPredicates(tag) } func (p *pathObject) CapitalizedSaveOutPredicates(tag string) *pathObject { return p.SaveOutPredicates(tag) } func (p *pathObject) CapitalizedLabelContext(call goja.FunctionCall) goja.Value { return p.LabelContext(call) } func (p *pathObject) CapitalizedFilter(args ...valFilter) (*pathObject, error) { return p.Filter(args...) } func (p *pathObject) CapitalizedLimit(limit int) *pathObject { return p.Limit(limit) } func (p *pathObject) CapitalizedSkip(offset int) *pathObject { return p.Skip(offset) } ================================================ FILE: query/graphql/graphql.go ================================================ package graphql import ( "context" "encoding/json" "fmt" "io" "io/ioutil" "strconv" "strings" "unicode" "github.com/dennwc/graphql/language/ast" "github.com/dennwc/graphql/language/lexer" "github.com/dennwc/graphql/language/parser" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" ) const Name = "graphql" // GraphQL charset: [_A-Za-z][_0-9A-Za-z]* // (https://facebook.github.io/graphql/#sec-Names) // IRI charset: [^#x00-#x20<>"{}|^`\] // (https://www.w3.org/TR/turtle/#grammar-production-IRIREF) func allowedNameRune(r rune) bool { // will include <> in the IRI value return r > 0x20 && !strings.ContainsRune("\"{}()|^`", r) && !unicode.IsSpace(r) } func init() { lexer.AllowNameRunes = allowedNameRune query.RegisterLanguage(query.Language{ Name: Name, Session: func(qs graph.QuadStore) query.Session { return NewSession(qs) }, HTTPError: httpError, HTTPQuery: httpQuery, }) } func NewSession(qs graph.QuadStore) *Session { return &Session{qs: qs} } type Session struct { qs graph.QuadStore } func (s *Session) Execute(ctx context.Context, qu string, opt query.Options) (query.Iterator, error) { switch opt.Collation { case query.Raw, query.JSON, query.REPL: default: return nil, &query.ErrUnsupportedCollation{Collation: opt.Collation} } q, err := Parse(strings.NewReader(qu)) if err != nil { return nil, err } return &results{ s: s, q: q, col: opt.Collation, }, nil } type results struct { s *Session q *Query col query.Collation res map[string]interface{} err error } func (it *results) Next(ctx context.Context) bool { if it.q == nil { return false } it.res, it.err = it.q.Execute(ctx, it.s.qs) it.q = nil return it.err == nil && len(it.res) != 0 } func (it *results) Result() interface{} { if len(it.res) == 0 { return nil } if it.col != query.REPL { return it.res } data, _ := json.MarshalIndent(it.res, "", " ") return string(data) } func (it *results) Err() error { return it.err } func (it *results) Close() error { it.q = nil return nil } // Configurable keywords and special field names. var ( ValueKey = "id" LimitKey = "first" SkipKey = "offset" AnyKey = "*" ) type Query struct { fields []field } type has struct { Via quad.IRI Rev bool Values []quad.Value Labels []quad.Value } type field struct { Via quad.IRI Alias string Rev bool Opt bool Labels []quad.Value Has []has Fields []field AllFields bool // fetch all fields UnNest bool // all fields will be saved to parent object } func (f field) isSave() bool { return len(f.Has)+len(f.Fields) == 0 && !f.AllFields } type object struct { id graph.Ref fields map[string]interface{} } func buildIterator(ctx context.Context, qs graph.QuadStore, p *path.Path) iterator.Shape { it, _ := p.BuildIterator(ctx).Optimize(ctx) return it } func iterateObject(ctx context.Context, qs graph.QuadStore, f *field, p *path.Path) (out []map[string]interface{}, _ error) { if len(f.Labels) != 0 { p = p.LabelContext(f.Labels) } else { p = p.LabelContext() } var ( limit = -1 skip = 0 ) for _, h := range f.Has { switch h.Via { case quad.IRI(ValueKey): // special key - "id" p = p.Is(h.Values...) case quad.IRI(LimitKey), quad.IRI(SkipKey): // limit and skip if len(h.Values) != 1 { return nil, fmt.Errorf("unexpected arguments: %v (%d)", h.Values, len(h.Values)) } n, ok := h.Values[0].(quad.Int) if !ok { return nil, fmt.Errorf("unexpected value type for %v: %T", string(h.Via), h.Values[0]) } if h.Via == quad.IRI(LimitKey) { limit = int(n) } else { skip = int(n) if skip < 0 { skip = 0 } } default: // everything else - Has constraint if len(h.Labels) != 0 { p = p.LabelContext(h.Labels) } if h.Rev { p = p.HasReverse(h.Via, h.Values...) } else { p = p.Has(h.Via, h.Values...) } if len(h.Labels) != 0 { p = p.LabelContext() } } } tail := func() { if skip > 0 { p = p.Skip(int64(skip)) } if limit >= 0 { p = p.Limit(int64(limit)) } } if f.AllFields { tail() it := buildIterator(ctx, qs, p).Iterate() defer it.Close() // we don't care about alternative paths to nodes here, so we will not call NextPath // and we haven't tagged anything, so we will not call TagResult either for i := 0; limit < 0 || i < limit; i++ { select { case <-ctx.Done(): return out, ctx.Err() default: } if !it.Next(ctx) { break } nv := it.Result() obj := make(map[string]interface{}) var err error obj[ValueKey], err = qs.NameOf(nv) if err != nil { return nil, err } sit := qs.QuadIterator(quad.Subject, nv).Iterate() for sit.Next(ctx) { q, err := qs.Quad(sit.Result()) if err != nil { sit.Close() return nil, err } if p, ok := q.Predicate.(quad.IRI); ok { obj[string(p)] = q.Object } else { obj[quad.ToString(q.Predicate)] = q.Object } } sit.Close() out = append(out, obj) } return out, it.Err() } unnest := make(map[string]bool) for _, f2 := range f.Fields { if f2.UnNest { unnest[f2.Alias] = true } if !f2.isSave() { continue } if f2.Via == quad.IRI(ValueKey) { p = p.Tag(f2.Alias) continue } if len(f2.Labels) != 0 { p = p.LabelContext(f2.Labels) } if f2.Opt { if f2.Rev { p = p.SaveOptionalReverse(f2.Via, f2.Alias) } else { p = p.SaveOptional(f2.Via, f2.Alias) } } else { if f2.Rev { p = p.SaveReverse(f2.Via, f2.Alias) } else { p = p.Save(f2.Via, f2.Alias) } } if len(f2.Labels) != 0 { p = p.LabelContext() } } tail() // first, collect result node ids and any tags associated with it (flat values) it := buildIterator(ctx, qs, p).Iterate() defer it.Close() var results []object for i := 0; limit < 0 || i < limit; i++ { select { case <-ctx.Done(): return out, ctx.Err() default: } if !it.Next(ctx) { break } fields := make(map[string][]graph.Ref) tags := make(map[string]graph.Ref) it.TagResults(tags) for k, v := range tags { fields[k] = []graph.Ref{v} } for it.NextPath(ctx) { select { case <-ctx.Done(): return out, ctx.Err() default: } tags = make(map[string]graph.Ref) it.TagResults(tags) dedup: for k, v := range tags { vals := fields[k] for _, v2 := range vals { if refs.ToKey(v) == refs.ToKey(v2) { continue dedup } } fields[k] = append(vals, v) } } obj := object{id: it.Result()} if len(fields) > 0 { obj.fields = make(map[string]interface{}, len(fields)) for k, arr := range fields { vals, err := graph.ValuesOf(ctx, qs, arr) if err != nil { return nil, err } if len(vals) == 1 { obj.fields[k] = vals[0] } else { obj.fields[k] = vals } } } results = append(results, obj) } if err := it.Err(); err != nil { return out, err } // next, load complex objects inside fields for _, r := range results { obj := r.fields if obj == nil { obj = make(map[string]interface{}) } for _, f2 := range f.Fields { if f2.isSave() { continue // skip flat values } // start from saved id for a field node p2 := path.StartPathNodes(qs, r.id) if len(f2.Labels) != 0 { p2 = p2.LabelContext(f2.Labels) } if f2.Rev { p2 = p2.In(f2.Via) } else { p2 = p2.Out(f2.Via) } if len(f2.Labels) != 0 { p2 = p2.LabelContext() } arr, err := iterateObject(ctx, qs, &f2, p2) if err != nil { return out, err } if f2.UnNest { if len(arr) > 1 { return nil, fmt.Errorf("cannot unnest more than one object on %q; use (%s: 1) to force", f2.Alias, LimitKey) } else if len(arr) == 0 { continue } for k, v := range arr[0] { obj[k] = v } } else { var v interface{} if len(arr) == 1 { v = arr[0] } else if len(arr) > 1 { v = arr } obj[f2.Alias] = v } } out = append(out, obj) } return out, nil } func (q *Query) Execute(ctx context.Context, qs graph.QuadStore) (map[string]interface{}, error) { out := make(map[string]interface{}) for _, f := range q.fields { arr, err := iterateObject(ctx, qs, &f, path.StartPath(qs)) if err != nil { return out, err } var v interface{} if len(arr) == 1 { v = arr[0] } else if len(arr) > 1 { v = arr } out[f.Alias] = v } return out, nil } func Parse(r io.Reader) (*Query, error) { data, err := ioutil.ReadAll(r) if err != nil { return nil, err } doc, err := parser.Parse(parser.ParseParams{Source: string(data)}) if err != nil { return nil, err } if len(doc.Definitions) != 1 { return nil, fmt.Errorf("unsupported query type") } def, ok := doc.Definitions[0].(*ast.OperationDefinition) if !ok { return nil, fmt.Errorf("unsupported query type: %T", doc.Definitions[0]) } else if def.Operation != "query" { return nil, fmt.Errorf("unsupported operation: %s", def.Operation) } fields, all, err := setToFields(def.SelectionSet, nil) if err != nil { return nil, err } else if all { return nil, fmt.Errorf("expand all is not supported at top level") } return &Query{fields: fields}, nil } func setToFields(set *ast.SelectionSet, labels []quad.Value) (out []field, all bool, _ error) { if set == nil { return } for _, s := range set.Selections { switch sel := s.(type) { case *ast.Field: fld, err := convField(sel, labels) if err != nil { return nil, false, err } if fld.Via == quad.IRI(AnyKey) { if len(set.Selections) != 1 { return nil, false, fmt.Errorf("expand all cannot be used with other fields") } else if len(fld.Has) != 0 || len(fld.Fields) != 0 { return nil, false, fmt.Errorf("filters inside expand all are not supported") } return nil, true, nil } out = append(out, fld) default: return nil, false, fmt.Errorf("unknown selection type: %T", s) } } return } func stringToVia(s string) (_ quad.IRI, rev bool) { if len(s) > 0 && s[0] == '~' { rev = true s = s[1:] } if len(s) > 2 && s[0] == '<' && s[len(s)-1] == '>' { s = s[1 : len(s)-1] } return quad.IRI(s), rev } func argsToHas(dst []has, args []*ast.Argument, rev bool, labels []quad.Value) (out []has, err error) { out = dst for _, arg := range args { var vals []quad.Value vals, err = convValue(arg.Value) if err != nil { return } h := has{Values: vals, Labels: labels} h.Via, h.Rev = stringToVia(arg.Name.Value) h.Rev = h.Rev != rev out = append(out, h) } return } func convField(fld *ast.Field, labels []quad.Value) (out field, err error) { out.Labels = labels name := fld.Name.Value if fld.Alias != nil && fld.Alias.Value != "" { out.Alias = fld.Alias.Value } else { out.Alias = name } out.Via, out.Rev = stringToVia(name) // first check for "label" directive - it will affect all traversals for _, d := range fld.Directives { if d.Name == nil { continue } switch d.Name.Value { case "label": if len(d.Arguments) == 0 { out.Labels = nil } else if len(d.Arguments) > 1 { return out, fmt.Errorf("label directive should have 0 or 1 argument") } else if a := d.Arguments[0]; a.Name == nil || a.Name.Value != "v" { return out, fmt.Errorf("label directive should have 'v' argument") } else { vals, err := convValue(a.Value) if err != nil { return out, fmt.Errorf("error parsing label: %v", err) } out.Labels = vals } } } for _, d := range fld.Directives { if d.Name == nil { continue } switch d.Name.Value { case "rev", "reverse": if len(d.Arguments) == 0 { out.Rev = out.Rev != true } else { out.Has, err = argsToHas(out.Has, d.Arguments, true, out.Labels) if err != nil { return } } case "opt", "optional": out.Opt = true case "label": // already processed case "unnest": out.UnNest = true default: return out, fmt.Errorf("unknown directive: %q", d.Name.Value) } } out.Fields, out.AllFields, err = setToFields(fld.SelectionSet, out.Labels) if err != nil { return } out.Has, err = argsToHas(out.Has, fld.Arguments, false, out.Labels) if err != nil { return } return } func convValue(v ast.Value) (out []quad.Value, _ error) { switch v := v.(type) { case *ast.EnumValue: s := v.Value if len(s) > 2 && s[0] == '<' && s[len(s)-1] == '>' { s = s[1 : len(s)-1] } if len(s) > 2 && s[0] == '_' && s[1] == ':' { return []quad.Value{quad.BNode(s[2:])}, nil } return []quad.Value{quad.IRI(s)}, nil case *ast.StringValue: return []quad.Value{quad.StringToValue(v.Value)}, nil case *ast.IntValue: pv, _ := strconv.Atoi(v.Value) return []quad.Value{quad.Int(pv)}, nil case *ast.FloatValue: pv, _ := strconv.ParseFloat(v.Value, 64) return []quad.Value{quad.Float(pv)}, nil case *ast.BooleanValue: return []quad.Value{quad.Bool(v.Value)}, nil case *ast.ListValue: for _, sv := range v.Values { cv, err := convValue(sv) if err != nil { return nil, err } else if len(cv) != 1 { return nil, fmt.Errorf("unexpected value array in list: %v (%d)", cv, len(cv)) } out = append(out, cv[0]) } return default: return nil, fmt.Errorf("unsupported value type: %T", v) } } ================================================ FILE: query/graphql/graphql_test.go ================================================ package graphql import ( "bytes" "context" "encoding/json" "reflect" "strings" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph/graphtest/testutil" "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc/rdf" ) func iris(arr ...string) (out []quad.Value) { for _, s := range arr { out = append(out, quad.IRI(s)) } return } var casesParse = []struct { query string expect []field }{ { `{ user(id: 3500401, http://iri: http://some_iri, follow: , n: _:bob, v: ["", "name", 3]) @rev(follow: "123"){ id: ` + ValueKey + `, type: ` + rdf.NS + "type" + `, followed: follow @reverse @label(v: ) { name: @label(v: ) followed: ~follow sname @label } isViewerFriend, profilePicture(size: 50) @unnest { uri, width @opt, height @rev } sub {*} } }`, []field{{ Via: "user", Alias: "user", Has: []has{ {"follow", true, []quad.Value{quad.String("123")}, nil}, {"id", false, []quad.Value{quad.Int(3500401)}, nil}, {"http://iri", false, iris("http://some_iri"), nil}, {"follow", false, iris("bob"), nil}, {"n", false, []quad.Value{quad.BNode("bob")}, nil}, {"v", false, []quad.Value{quad.IRI("bob"), quad.String("name"), quad.Int(3)}, nil}, }, Fields: []field{ {Via: quad.IRI(ValueKey), Alias: "id"}, {Via: quad.IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), Alias: "type"}, { Via: "follow", Alias: "followed", Rev: true, Labels: iris("fb"), Fields: []field{ {Via: "name", Alias: "name", Labels: iris("google")}, {Via: "follow", Alias: "followed", Rev: true, Labels: iris("fb")}, {Via: "sname", Alias: "sname"}, }, }, {Via: "isViewerFriend", Alias: "isViewerFriend"}, { Via: "profilePicture", Alias: "profilePicture", Has: []has{{"size", false, []quad.Value{quad.Int(50)}, nil}}, Fields: []field{ {Via: "uri", Alias: "uri"}, {Via: "width", Alias: "width", Opt: true}, {Via: "height", Alias: "height", Rev: true}, }, UnNest: true, }, {Via: "sub", Alias: "sub", AllFields: true}, }, }}, }, } func TestParse(t *testing.T) { for _, c := range casesParse { q, err := Parse(strings.NewReader(c.query)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.fields, c.expect) { t.Fatalf("\n%#v\nvs\n%#v", q.fields, c.expect) } } } type M = map[string]interface{} var casesExecute = []struct { name string query string result M }{ { "cool people and friends", `{ me(status: "cool_person") { id: ` + ValueKey + ` follows { ` + ValueKey + ` status } followed: follows @rev { ` + ValueKey + ` } } }`, M{ "me": []M{ { "id": quad.IRI("bob"), "follows": nil, "followed": []M{ {ValueKey: quad.IRI("alice")}, {ValueKey: quad.IRI("dani")}, {ValueKey: quad.IRI("charlie")}, }, }, { "id": quad.IRI("dani"), "follows": []M{ { ValueKey: quad.IRI("bob"), "status": quad.String("cool_person"), }, { ValueKey: quad.IRI("greg"), "status": []quad.Value{ quad.String("cool_person"), quad.String("smart_person"), }, }, }, "followed": M{ ValueKey: quad.IRI("charlie"), }, }, { "id": quad.IRI("greg"), "follows": nil, "followed": []M{ {ValueKey: quad.IRI("dani")}, {ValueKey: quad.IRI("fred")}, }, }, }, }, }, { "skip and limit", `{ me(status: "cool_person", ` + LimitKey + `: 1, ` + SkipKey + `: 1) { id: ` + ValueKey + ` follows(` + LimitKey + `: 1) @opt { ` + ValueKey + ` } } }`, M{ "me": M{ "id": quad.IRI("dani"), "follows": M{ ValueKey: quad.IRI("bob"), }, }, }, }, { "labels", `{ me { id: ` + ValueKey + ` status @label(v: ) } }`, M{ "me": []M{ { "id": quad.IRI("emily"), "status": quad.String("smart_person"), }, { "id": quad.IRI("greg"), "status": quad.String("smart_person"), }, }, }, }, { "expand all", `{ me { id: ` + ValueKey + ` status @label(v: ) follows {*} } }`, M{ "me": []M{ { "id": quad.IRI("emily"), "status": quad.String("smart_person"), "follows": M{ "id": quad.IRI("fred"), "follows": quad.IRI("greg"), }, }, { "id": quad.IRI("greg"), "status": quad.String("smart_person"), "follows": nil, }, }, }, }, { "unnest object", `{ me(id: fred) { id: ` + ValueKey + ` follows @unnest { friend: ` + ValueKey + ` friend_status: status followed: follows(` + LimitKey + `: 1) @rev @unnest { fof: ` + ValueKey + ` } } } }`, M{ "me": M{ "id": quad.IRI("fred"), "fof": quad.IRI("dani"), "friend": quad.IRI("greg"), "friend_status": []quad.Value{ quad.String("cool_person"), quad.String("smart_person"), }, }, }, }, { "unnest object (non existent)", `{ me(id: fred) { id: ` + ValueKey + ` follows_missing @unnest { friend: ` + ValueKey + ` friend_status: status } } }`, M{ "me": M{ "id": quad.IRI("fred"), }, }, }, { "all optional", `{ nodes { id, status @opt } }`, M{ "nodes": []M{ {"id": quad.IRI("alice")}, {"id": quad.IRI("follows")}, {"id": quad.IRI("bob"), "status": quad.String("cool_person")}, {"id": quad.IRI("fred")}, {"id": quad.IRI("status")}, {"id": quad.String("cool_person")}, {"id": quad.IRI("dani"), "status": quad.String("cool_person")}, {"id": quad.IRI("charlie")}, {"id": quad.IRI("greg"), "status": []quad.Value{ quad.String("cool_person"), quad.String("smart_person"), }}, {"id": quad.IRI("emily"), "status": quad.String("smart_person")}, {"id": quad.IRI("predicates")}, {"id": quad.IRI("are")}, {"id": quad.String("smart_person")}, {"id": quad.IRI("smart_graph")}, }, }, }, } func toJSON(o interface{}) string { buf := bytes.NewBuffer(nil) json.NewEncoder(buf).Encode(o) buf2 := bytes.NewBuffer(nil) json.Indent(buf2, buf.Bytes(), "", " ") return buf2.String() } func TestExecute(t *testing.T) { qs := memstore.New() qw := testutil.MakeWriter(t, qs, nil) quads := testutil.LoadGraph(t, "../../data/testdata.nq") err := qw.AddQuadSet(quads) require.NoError(t, err) for _, c := range casesExecute { t.Run(c.name, func(t *testing.T) { q, err := Parse(strings.NewReader(c.query)) require.NoError(t, err) out, err := q.Execute(context.Background(), qs) require.NoError(t, err) require.Equal(t, c.result, out, "results:\n%v\n\nvs\n\n%v", toJSON(c.result), toJSON(out)) }) } } ================================================ FILE: query/graphql/http.go ================================================ package graphql import ( "context" "encoding/json" "io" "github.com/dennwc/graphql/gqlerrors" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query" ) type httpResult struct { Data interface{} `json:"data"` Errors []gqlerrors.FormattedError `json:"errors,omitempty"` } func httpError(w query.ResponseWriter, err error) { json.NewEncoder(w).Encode(httpResult{ Errors: []gqlerrors.FormattedError{{ Message: err.Error(), }}, }) } func httpQuery(ctx context.Context, qs graph.QuadStore, w query.ResponseWriter, r io.Reader) { q, err := Parse(r) if err != nil { httpError(w, err) return } m, err := q.Execute(ctx, qs) if err != nil { httpError(w, err) return } json.NewEncoder(w).Encode(httpResult{Data: m}) } ================================================ FILE: query/linkedql/entity_identfier.go ================================================ package linkedql import ( "encoding/json" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) // EntityIdentifierI is an interface to be used where a single entity identifier is expected. type EntityIdentifierI interface { BuildIdentifier(ns *voc.Namespaces) (quad.Value, error) } // EntityIdentifier is a struct wrapping the interface EntityIdentifierI type EntityIdentifier struct { EntityIdentifierI } // NewEntityIdentifier constructs a new EntityIdentifer from a EntityIdentiferI func NewEntityIdentifier(v EntityIdentifierI) EntityIdentifier { return EntityIdentifier{EntityIdentifierI: v} } // UnmarshalJSON implements RawMessage func (p *EntityIdentifier) UnmarshalJSON(data []byte) error { var errors []error var iri EntityIRI err := json.Unmarshal(data, &iri) if err == nil { p.EntityIdentifierI = iri return nil } errors = append(errors, err) var bnode EntityBNode err = json.Unmarshal(data, &bnode) if err == nil { p.EntityIdentifierI = bnode return nil } errors = append(errors, err) var s EntityIdentifierString err = json.Unmarshal(data, &s) if err == nil { p.EntityIdentifierI = s return nil } errors = append(errors, err) return formatMultiError(errors) } // EntityIRI is an entity IRI. type EntityIRI quad.IRI // BuildIdentifier implements EntityIdentifier func (iri EntityIRI) BuildIdentifier(ns *voc.Namespaces) (quad.Value, error) { return quad.IRI(iri).FullWith(ns), nil } // EntityBNode is an entity BNode. type EntityBNode quad.BNode // BuildIdentifier implements EntityIdentifier func (i EntityBNode) BuildIdentifier(ns *voc.Namespaces) (quad.Value, error) { return quad.BNode(i), nil } // EntityIdentifierString is an entity IRI or BNode strings. type EntityIdentifierString string // BuildIdentifier implements EntityIdentifier func (i EntityIdentifierString) BuildIdentifier(ns *voc.Namespaces) (quad.Value, error) { identifier, err := parseIdentifier(string(i)) if err != nil { return nil, err } return AbsoluteValue(identifier, ns), nil } ================================================ FILE: query/linkedql/errors.go ================================================ package linkedql import "fmt" func formatMultiError(errors []error) error { joinedErr := "" for _, err := range errors { joinedErr += "; " + err.Error() } return fmt.Errorf("Could not parse PropertyPath: %v", joinedErr) } ================================================ FILE: query/linkedql/graph_pattern.go ================================================ package linkedql // GraphPattern represents a JSON-LD document type GraphPattern = map[string]interface{} ================================================ FILE: query/linkedql/iter_docs.go ================================================ package linkedql import ( "context" "github.com/cayleygraph/cayley/query" "github.com/piprate/json-gold/ld" ) var ( _ query.Iterator = (*DocumentIterator)(nil) ) // DocumentIterator is an iterator of documents from the graph type DocumentIterator struct { tagsIt *TagsIterator dataset *ld.RDFDataset err error exhausted bool } // NewDocumentIterator returns a new DocumentIterator for a QuadStore and Path. func NewDocumentIterator(valueIt *ValueIterator) *DocumentIterator { tagsIt := &TagsIterator{ValueIt: valueIt, Selected: nil} return &DocumentIterator{tagsIt: tagsIt, exhausted: false} } func (it *DocumentIterator) getDataset(ctx context.Context) (*ld.RDFDataset, error) { d := ld.NewRDFDataset() for it.tagsIt.Next(ctx) { r := it.tagsIt.ValueIt.scanner.Result() if err := it.tagsIt.Err(); err != nil { if err != nil { return nil, err } } if r == nil { continue } err := it.tagsIt.addResultsToDataset(d, r) if err != nil { return nil, err } } return d, nil } // Next implements query.Iterator. func (it *DocumentIterator) Next(ctx context.Context) bool { if !it.exhausted { d, err := it.getDataset(ctx) if err != nil { it.err = err } else { it.dataset = d } it.exhausted = true return true } return false } // Result implements query.Iterator. func (it *DocumentIterator) Result() interface{} { context := make(map[string]interface{}) opts := ld.NewJsonLdOptions("") c, err := datasetToCompact(it.dataset, context, opts) if err != nil { it.err = err } return c } // Err implements query.Iterator. func (it *DocumentIterator) Err() error { if it.tagsIt == nil { return nil } if it.err != nil { return it.err } return it.tagsIt.Err() } // Close implements query.Iterator. func (it *DocumentIterator) Close() error { if it.tagsIt == nil { return nil } return it.tagsIt.Close() } ================================================ FILE: query/linkedql/iter_tags.go ================================================ package linkedql import ( "context" "fmt" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/jsonld" "github.com/piprate/json-gold/ld" ) var _ query.Iterator = (*TagsIterator)(nil) // TagsIterator is a result iterator for records consisting of selected tags // or all the tags in the query. type TagsIterator struct { ValueIt *ValueIterator Selected []string ExcludeID bool err error } // NewTagsIterator creates a new TagsIterator func NewTagsIterator(valueIt *ValueIterator, selected []string, excludeID bool) TagsIterator { return TagsIterator{ ValueIt: valueIt, Selected: selected, ExcludeID: excludeID, err: nil, } } // Next implements query.Iterator. func (it *TagsIterator) Next(ctx context.Context) bool { return it.ValueIt.Next(ctx) } func (it *TagsIterator) addQuadFromRef(dataset *ld.RDFDataset, subject ld.Node, tag string, ref refs.Ref) error { p := ld.NewIRI(tag) rname, err := it.ValueIt.Namer.NameOf(ref) if err != nil { return err } o, err := jsonld.ToNode(rname) if err != nil { return err } q := ld.NewQuad(subject, p, o, "") dataset.Graphs["@default"] = append(dataset.Graphs["@default"], q) return nil } func toSubject(namer refs.Namer, result refs.Ref) (ld.Node, error) { v, err := namer.NameOf(result) if err != nil { return nil, err } id, ok := v.(quad.Identifier) if !ok { return nil, fmt.Errorf("Expected subject to be an entity identifier but instead received: %v", v) } return jsonld.ToNode(id) } func (it *TagsIterator) addResultsToDataset(dataset *ld.RDFDataset, result refs.Ref) error { s, err := toSubject(it.ValueIt.Namer, result) if err != nil { return err } refTags := make(map[string]refs.Ref) it.ValueIt.scanner.TagResults(refTags) if len(it.Selected) == 0 { for tag, ref := range refTags { it.addQuadFromRef(dataset, s, tag, ref) } } else { for _, tag := range it.Selected { it.addQuadFromRef(dataset, s, tag, refTags[tag]) } } return nil } // Result implements query.Iterator. func (it *TagsIterator) Result() interface{} { // FIXME(iddan): only convert when collation is JSON/JSON-LD, leave as Ref otherwise r := it.ValueIt.scanner.Result() if r == nil { return nil } d := ld.NewRDFDataset() err := it.addResultsToDataset(d, r) if err != nil { it.err = err return nil } doc, err := singleDocumentFromRDF(d) if err != nil { it.err = err return nil } if !it.ExcludeID { m := doc.(map[string]interface{}) delete(m, "@id") return m } return doc } // Err implements query.Iterator. func (it *TagsIterator) Err() error { if it.err != nil { return it.err } return it.ValueIt.Err() } // Close implements query.Iterator. func (it *TagsIterator) Close() error { return it.ValueIt.Close() } ================================================ FILE: query/linkedql/iter_tags_test.go ================================================ package linkedql import ( "fmt" "testing" "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/quad" "github.com/piprate/json-gold/ld" "github.com/stretchr/testify/require" ) var ( namespace = "http://example.com/" alice = namespace + "alice" likes = namespace + "likes" blank = quad.RandomBlankNode() name = namespace + "name" aliceName = quad.String("Alice") aliceLikesBlank = quad.Quad{ Subject: quad.IRI(alice), Predicate: quad.IRI(likes), Object: blank, } aliceNameAlice = quad.Quad{ Subject: quad.IRI(alice), Predicate: quad.IRI(name), Object: aliceName, } ) var testCases = []struct { name string data quad.Quad value quad.Value expected ld.Node err error }{ { name: "Success for IRI", data: aliceLikesBlank, value: aliceLikesBlank.Subject, expected: ld.NewIRI(alice), err: nil, }, { name: "Success for Blank Node", data: aliceLikesBlank, value: aliceLikesBlank.Object, expected: ld.NewBlankNode(string(blank)), err: nil, }, { name: "Failure for String", data: aliceNameAlice, value: aliceNameAlice.Object, expected: nil, err: fmt.Errorf("Expected subject to be an entity identifier but instead received: %v", aliceName), }, } func TestToSubject(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { store := memstore.New(testCase.data) r, err := store.ValueOf(testCase.value) require.NoError(t, err) s, err := toSubject(store, r) if testCase.err == nil { require.NoError(t, err) require.Equal(t, testCase.expected, s) } else { require.Equal(t, testCase.err, err) } }) } } ================================================ FILE: query/linkedql/iter_values.go ================================================ package linkedql import ( "context" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/jsonld" "github.com/cayleygraph/quad/voc" ) var _ query.Iterator = (*ValueIterator)(nil) // ValueIterator is an iterator of values from the graph. type ValueIterator struct { Namer refs.Namer path *path.Path scanner iterator.Scanner err error } // NewValueIterator returns a new ValueIterator for a path and namer. func NewValueIterator(p *path.Path, namer refs.Namer) *ValueIterator { return &ValueIterator{Namer: namer, path: p} } // NewValueIteratorFromPathStep attempts to build a path from PathStep and return a new ValueIterator of it. // If BuildPath fails returns error. func NewValueIteratorFromPathStep(step PathStep, qs graph.QuadStore, ns *voc.Namespaces) (*ValueIterator, error) { p, err := step.BuildPath(qs, ns) if err != nil { return nil, err } return NewValueIterator(p, qs), nil } // Next implements query.Iterator. func (it *ValueIterator) Next(ctx context.Context) bool { if it.scanner == nil { it.scanner = it.path.BuildIterator(ctx).Iterate() } return it.scanner.Next(ctx) } // Value returns the current value func (it *ValueIterator) Value() quad.Value { if it.scanner == nil { return nil } rname, err := it.Namer.NameOf(it.scanner.Result()) if err != nil { it.err = err } return rname } // Result implements query.Iterator. func (it *ValueIterator) Result() interface{} { // FIXME(iddan): only convert when collation is JSON/JSON-LD, leave as Ref otherwise return jsonld.FromValue(it.Value()) } // Err implements query.Iterator. func (it *ValueIterator) Err() error { if it.err != nil { return it.err } if it.scanner == nil { return nil } return it.scanner.Err() } // Close implements query.Iterator. func (it *ValueIterator) Close() error { if it.scanner == nil { return nil } return it.scanner.Close() } ================================================ FILE: query/linkedql/jsonld_util.go ================================================ package linkedql import ( "fmt" "github.com/piprate/json-gold/ld" ) // datasetToCompact transforms a RDF dataset to a compact JSON-LD document. func datasetToCompact(dataset *ld.RDFDataset, context interface{}, opts *ld.JsonLdOptions) (interface{}, error) { api := ld.NewJsonLdApi() proc := ld.NewJsonLdProcessor() d, err := api.FromRDF(dataset, opts) if err != nil { return nil, err } c, err := proc.Compact(d, context, opts) if err != nil { return nil, err } return c, nil } // singleDocumentFromRDF transforms a RDF dataset to a single map JSON-LD document. func singleDocumentFromRDF(dataset *ld.RDFDataset) (interface{}, error) { api := ld.NewJsonLdApi() opts := ld.NewJsonLdOptions("") documents, err := api.FromRDF(dataset, opts) if err != nil { return nil, err } if len(documents) != 1 { return nil, fmt.Errorf("Unexpected number of documents") } return documents[0], nil } ================================================ FILE: query/linkedql/linkedql.go ================================================ package linkedql import ( "context" "errors" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/quad/voc" ) const ( // Name is the name exposed to the query interface. Name = "linkedql" // Namespace is an RDF namespace used for LinkedQL classes. Namespace = "http://cayley.io/linkedql#" // Prefix is an RDF namespace prefix used for LinkedQL classes. Prefix = "linkedql:" ) func init() { // IRI namespace support voc.Register(voc.Namespace{Full: Namespace, Prefix: Prefix}) // register the language query.RegisterLanguage(query.Language{ Name: Name, Session: func(qs graph.QuadStore) query.Session { return NewSession(qs) }, }) } var _ query.Session = &Session{} // Session represents a LinkedQL query processing. type Session struct { qs graph.QuadStore } // NewSession creates a new Session. func NewSession(qs graph.QuadStore) *Session { return &Session{ qs: qs, } } // Execute for a given context, query and options return an iterator of results. func (s *Session) Execute(ctx context.Context, query string, opt query.Options) (query.Iterator, error) { item, err := Unmarshal([]byte(query)) if err != nil { return nil, err } ns := voc.Namespaces{} step, ok := item.(Step) if !ok { return nil, errors.New("must execute a Step") } return BuildIterator(step, s.qs, &ns) } // BuildIterator for given Step returns a query.Iterator func BuildIterator(step Step, qs graph.QuadStore, ns *voc.Namespaces) (query.Iterator, error) { switch s := step.(type) { case IteratorStep: return s.BuildIterator(qs, ns) case PathStep: return NewValueIteratorFromPathStep(s, qs, ns) } return nil, errors.New("must execute a IteratorStep or PathStep") } ================================================ FILE: query/linkedql/property_path.go ================================================ package linkedql import ( "encoding/json" "fmt" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) // PropertyPathI is an interface to be used where a path of properties is expected. type PropertyPathI interface { BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) } // PropertyPath is a struct wrapping PropertyPathI type PropertyPath struct { PropertyPathI } // NewPropertyPath constructs a new PropertyPath func NewPropertyPath(p PropertyPathI) *PropertyPath { return &PropertyPath{PropertyPathI: p} } // Description implements Step. func (*PropertyPath) Description() string { return "PropertyPath is a string, multiple strins or path describing a set of properties" } // UnmarshalJSON implements RawMessage func (p *PropertyPath) UnmarshalJSON(data []byte) error { var errors []error var propertyIRIs PropertyIRIs err := json.Unmarshal(data, &propertyIRIs) if err == nil { p.PropertyPathI = propertyIRIs return nil } errors = append(errors, err) var propertyIRIStrings PropertyIRIStrings err = json.Unmarshal(data, &propertyIRIStrings) if err == nil { p.PropertyPathI = propertyIRIStrings return nil } errors = append(errors, err) var propertyIRI PropertyIRI err = json.Unmarshal(data, &propertyIRI) if err == nil { p.PropertyPathI = propertyIRI return nil } errors = append(errors, err) var propertyIRIString PropertyIRIString err = json.Unmarshal(data, &propertyIRIString) if err == nil { p.PropertyPathI = propertyIRIString return nil } errors = append(errors, err) step, err := Unmarshal(data) if err == nil { pathStep, ok := step.(PathStep) if ok { p.PropertyPathI = pathStep return nil } errors = append(errors, fmt.Errorf("Step of type %T is not a PathStep. A PropertyPath step must be a PathStep", step)) } errors = append(errors, err) return formatMultiError(errors) } // PropertyIRIs is a slice of property IRIs. type PropertyIRIs []PropertyIRI // BuildPath implements PropertyPath. func (p PropertyIRIs) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { var values []quad.Value for _, iri := range p { values = append(values, iri.full(ns)) } return path.StartPath(qs, values...), nil } // PropertyIRIStrings is a slice of property IRI strings. type PropertyIRIStrings []string // PropertyIRIs casts PropertyIRIStrings into PropertyIRIs func (p PropertyIRIStrings) PropertyIRIs() PropertyIRIs { var iris PropertyIRIs for _, iri := range p { iris = append(iris, PropertyIRI(iri)) } return iris } // BuildPath implements PropertyPath. func (p PropertyIRIStrings) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { return p.PropertyIRIs().BuildPath(qs, ns) } // PropertyIRI is an IRI of a Property type PropertyIRI quad.IRI func (p PropertyIRI) full(ns *voc.Namespaces) quad.IRI { return quad.IRI(p).FullWith(ns) } // BuildPath implements PropertyPath func (p PropertyIRI) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { return path.StartPath(qs, p.full(ns)), nil } // PropertyIRIString is a string of IRI of a Property type PropertyIRIString string // BuildPath implements PropertyPath func (p PropertyIRIString) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { iri := PropertyIRI(p) return iri.BuildPath(qs, ns) } // PropertyStep is a step that should resolve to a path of properties type PropertyStep struct { PathStep } // BuildPath implements PropertyPath func (p PropertyStep) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { return p.BuildPath(qs, ns) } ================================================ FILE: query/linkedql/registry.go ================================================ package linkedql import ( "encoding/json" "fmt" "reflect" "strings" "github.com/cayleygraph/quad" "github.com/piprate/json-gold/ld" ) var ( typeByName = make(map[string]reflect.Type) nameByType = make(map[reflect.Type]string) ) // TypeByName returns a type by its registration name. See Register. func TypeByName(name string) (reflect.Type, bool) { t, ok := typeByName[name] return t, ok } // RegisteredTypes returns type names of all registered types. func RegisteredTypes() []string { out := make([]string, 0, len(typeByName)) for k := range typeByName { out = append(out, k) } return out } // RegistryItem in the registry. type RegistryItem interface { Description() string } // Register adds an Item type to the registry. func Register(typ RegistryItem) { tp := reflect.TypeOf(typ) if tp.Kind() == reflect.Ptr { tp = tp.Elem() } if tp.Kind() != reflect.Struct { panic("only structs are allowed") } name := Namespace + tp.Name() if _, ok := typeByName[name]; ok { panic("this name was already registered") } typeByName[name] = tp nameByType[tp] = name } var ( graphPattern = reflect.TypeOf(GraphPattern(nil)) quadValue = reflect.TypeOf((*quad.Value)(nil)).Elem() quadSliceValue = reflect.TypeOf([]quad.Value{}) quadIRI = reflect.TypeOf(quad.IRI("")) quadSliceIRI = reflect.TypeOf([]quad.IRI{}) ) // Unmarshal attempts to unmarshal an Item or returns error. func Unmarshal(data []byte) (RegistryItem, error) { // TODO: make it a part of quad/jsonld package. data, err := normalizeQuery(data) if err != nil { return nil, err } var m map[string]json.RawMessage if err := json.Unmarshal(data, &m); err != nil { return nil, err } var typ string if err := json.Unmarshal(m["@type"], &typ); err != nil { return nil, err } delete(m, "@type") tp, ok := TypeByName(typ) if !ok { return nil, fmt.Errorf("unsupported item: %q", typ) } item := reflect.New(tp).Elem() for i := 0; i < tp.NumField(); i++ { f := tp.Field(i) name := f.Name tag := strings.SplitN(f.Tag.Get("json"), ",", 2)[0] if tag == "-" { continue } else if tag != "" { name = Namespace + tag } v, ok := m[name] if !ok { continue } fv := item.Field(i) switch f.Type { case graphPattern: var a interface{} err := json.Unmarshal(v, &a) if err != nil { return nil, err } fv.Set(reflect.ValueOf(a)) continue case quadValue: var a interface{} err := json.Unmarshal(v, &a) if err != nil { return nil, err } value, err := parseValue(a) if err != nil { return nil, err } fv.Set(reflect.ValueOf(value)) continue case quadSliceValue: var a interface{} err := json.Unmarshal(v, &a) if err != nil { return nil, err } arr, ok := a.([]interface{}) if !ok { arr = []interface{}{a} } var values []quad.Value for _, item := range arr { value, err := parseValue(item) if err != nil { return nil, err } values = append(values, value) } fv.Set(reflect.ValueOf(values)) continue case quadIRI: var a interface{} err := json.Unmarshal(v, &a) if err != nil { return nil, err } s, ok := a.(string) if !ok { return nil, fmt.Errorf("Expected a string but received %v instead", a) } val, err := parseIRI(s) if err != nil { return nil, err } fv.Set(reflect.ValueOf(val)) continue case quadSliceIRI: var a interface{} err := json.Unmarshal(v, &a) if err != nil { return nil, err } arr, ok := a.([]interface{}) if !ok { arr = []interface{}{a} } var values []quad.IRI for _, item := range arr { s, ok := item.(string) if !ok { return nil, fmt.Errorf("Expected a string but received %v instead", item) } val, err := parseIRI(s) if err != nil { return nil, err } values = append(values, val) } fv.Set(reflect.ValueOf(values)) continue } switch f.Type.Kind() { case reflect.Interface: s, err := Unmarshal(v) if err != nil { return nil, err } fv.Set(reflect.ValueOf(s)) case reflect.Slice: el := f.Type.Elem() if el.Kind() != reflect.Interface { err := json.Unmarshal(v, fv.Addr().Interface()) if err != nil { return nil, err } } else { var arr []json.RawMessage err := json.Unmarshal(v, &arr) if err != nil { var i json.RawMessage iErr := json.Unmarshal(v, &i) if iErr != nil { return nil, err } arr = []json.RawMessage{i} } if arr != nil { va := reflect.MakeSlice(f.Type, len(arr), len(arr)) for i, v := range arr { s, err := Unmarshal(v) if err != nil { return nil, err } va.Index(i).Set(reflect.ValueOf(s)) } fv.Set(va) } } default: err := json.Unmarshal(v, fv.Addr().Interface()) if err != nil { return nil, err } } } return item.Addr().Interface().(RegistryItem), nil } func normalizeQuery(data []byte) ([]byte, error) { var query interface{} json.Unmarshal(data, &query) processor := ld.NewJsonLdProcessor() opts := ld.NewJsonLdOptions("") compact, err := processor.Compact(query, nil, opts) if err != nil { return nil, err } return json.Marshal(compact) } func parseBNode(s string) (quad.BNode, error) { if !strings.HasPrefix(s, "_:") { return "", fmt.Errorf("blank node ID must start with \"_:\"") } return quad.BNode(s[2:]), nil } func parseIRI(s string) (quad.IRI, error) { return quad.IRI(s), nil } func parseIdentifier(s string) (quad.Value, error) { bnode, err := parseBNode(s) if err == nil { return bnode, nil } iri, err := parseIRI(s) if err == nil { return iri, nil } return nil, fmt.Errorf("can not parse JSON-LD identifier: %#v", s) } func parseIdentifierString(a interface{}) (string, error) { m, ok := a.(map[string]interface{}) if !ok { return "", fmt.Errorf("unexpected type: %T", a) } id, ok := m["@id"].(string) if !ok { return "", fmt.Errorf("expected a @id key") } return id, nil } func parseLiteral(a interface{}) (quad.Value, error) { switch a := a.(type) { case string: return quad.String(a), nil case int64: return quad.Int(a), nil case float64: i := int64(a) if a == float64(i) { return quad.Int(i), nil } return quad.Float(a), nil case bool: return quad.Bool(a), nil case map[string]interface{}: if val, ok := a["@value"].(string); ok { if lang, ok := a["@language"].(string); ok { return quad.LangString{Value: quad.String(val), Lang: lang}, nil } if typ, ok := a["@type"].(string); ok { return quad.TypedString{Value: quad.String(val), Type: quad.IRI(typ)}, nil } } } return nil, fmt.Errorf("can not parse %#v as a literal", a) } func parseValue(a interface{}) (quad.Value, error) { identifierString, err := parseIdentifierString(a) if err == nil { identifier, err := parseIdentifier(identifierString) if err == nil { return identifier, nil } } lit, err := parseLiteral(a) if err == nil { return lit, nil } return nil, fmt.Errorf("can not parse JSON-LD value: %#v", a) } ================================================ FILE: query/linkedql/registry_test.go ================================================ package linkedql import ( "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" "github.com/stretchr/testify/require" ) func init() { Register(&TestStep{}) } var unmarshalCases = []struct { name string data string exp Step }{ { name: "simple", data: `{ "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "TestStep", "limit": 10 }`, exp: &TestStep{Limit: 10}, }, { name: "simple", data: `{ "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "TestStep", "tags": ["a", "b"] }`, exp: &TestStep{Tags: []string{"a", "b"}}, }, { name: "nested", data: `{ "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "TestStep", "limit": 10, "from": { "@type": "TestStep", "limit": 15, "from": { "@type": "TestStep", "limit": 20 } } }`, exp: &TestStep{ Limit: 10, From: &TestStep{ Limit: 15, From: &TestStep{ Limit: 20, }, }, }, }, { name: "nested slice", data: `{ "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "TestStep", "limit": 10, "sub": [ { "@type": "TestStep", "limit": 15 }, { "@type": "TestStep", "limit": 20 } ] }`, exp: &TestStep{ Limit: 10, Sub: []PathStep{ &TestStep{ Limit: 15, }, &TestStep{ Limit: 20, }, }, }, }, } type TestStep struct { Limit int `json:"limit"` Tags []string `json:"tags"` From PathStep `json:"from"` Sub []PathStep `json:"sub"` } func (s *TestStep) Description() string { return "A TestStep for checking the registry" } func (s *TestStep) BuildIterator(qs graph.QuadStore, ns *voc.Namespaces) (query.Iterator, error) { panic("Can't build iterator for TestStep") } func (s *TestStep) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { panic("Can't build path for TestStep") } func TestUnmarshalStep(t *testing.T) { for _, c := range unmarshalCases { t.Run(c.name, func(t *testing.T) { s, err := Unmarshal([]byte(c.data)) require.NoError(t, err) require.Equal(t, c.exp, s) }) } } ================================================ FILE: query/linkedql/step_types.go ================================================ package linkedql import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) // Step is a logical part in the query type Step interface { RegistryItem } // IteratorStep is a step that can build an Iterator. type IteratorStep interface { Step BuildIterator(qs graph.QuadStore, ns *voc.Namespaces) (query.Iterator, error) } // PathStep is a Step that can build a Path. type PathStep interface { Step BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) } ================================================ FILE: query/linkedql/steps/as.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&As{}) } var _ linkedql.PathStep = (*As)(nil) // As corresponds to .tag(). type As struct { From linkedql.PathStep `json:"from"` Name string `json:"name"` } // Description implements Step. func (s *As) Description() string { return "assigns the resolved values of the from step to a given name. The name can be used with the Select and Documents steps to retrieve the values or to return to the values in further steps with the Back step. It resolves to the values of the from step." } // BuildPath implements linkedql.PathStep. func (s *As) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Tag(s.Name), nil } ================================================ FILE: query/linkedql/steps/back.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Back{}) } var _ linkedql.PathStep = (*Back)(nil) // Back corresponds to .back(). type Back struct { From linkedql.PathStep `json:"from"` Name string `json:"name"` } // Description implements Step. func (s *Back) Description() string { return "resolves to the values of the previous the step or the values assigned to name in a former step." } // BuildPath implements linkedql.PathStep. func (s *Back) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Back(s.Name), nil } ================================================ FILE: query/linkedql/steps/both.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Both{}) } var _ linkedql.PathStep = (*Both)(nil) // Both corresponds to .both(). type Both struct { From linkedql.PathStep `json:"from"` Properties *linkedql.PropertyPath `json:"properties"` } // Description implements Step. func (s *Both) Description() string { return "is like View but resolves to both the object values and references to the values of the given properties in via. It is the equivalent for the Union of View and ViewReverse of the same property." } // BuildPath implements linkedql.PathStep. func (s *Both) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } viaPath, err := s.Properties.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Both(viaPath), nil } ================================================ FILE: query/linkedql/steps/collect.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Collect{}) } var _ linkedql.PathStep = (*Collect)(nil) // Collect corresponds to .view(). type Collect struct { From linkedql.PathStep `json:"from"` Name quad.IRI `json:"name"` } // Description implements Step. func (s *Collect) Description() string { return "Recursively resolves values of a list (also known as RDF collection)" } var ( first = quad.IRI("rdf:first").Full() rest = quad.IRI("rdf:rest").Full() rdfNil = quad.IRI("rdf:nil").Full() ) // BuildPath implements linkedql.PathStep. func (s *Collect) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } p := fromPath. Out(s.Name). Save(first, string(first)). Save(rest, string(rest)). Or( fromPath.Out(s.Name).FollowRecursive(rest, -1, nil). Save(first, string(first)). Save(rest, string(rest)), ). Or(fromPath.Save(s.Name, string(s.Name))) return p, nil } ================================================ FILE: query/linkedql/steps/count.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Count{}) } var _ linkedql.PathStep = (*Count)(nil) // Count corresponds to .count(). type Count struct { From linkedql.PathStep `json:"from"` } // Description implements Step. func (s *Count) Description() string { return "resolves to the number of the resolved values of the from step" } // BuildPath implements linkedql.PathStep. func (s *Count) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Count(), nil } ================================================ FILE: query/linkedql/steps/difference.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Difference{}) } var _ linkedql.PathStep = (*Difference)(nil) // Difference corresponds to .difference(). type Difference struct { From linkedql.PathStep `json:"from"` Steps []linkedql.PathStep `json:"steps"` } // Description implements Step. func (s *Difference) Description() string { return "resolves to all the values resolved by the from step different then the values resolved by the provided steps. Caution: it might be slow to execute." } // BuildPath implements linkedql.PathStep. func (s *Difference) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } path := fromPath for _, step := range s.Steps { p, err := step.BuildPath(qs, ns) if err != nil { return nil, err } path = path.Except(p) } return path, nil } ================================================ FILE: query/linkedql/steps/greater_than.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&GreaterThan{}) } var _ linkedql.PathStep = (*GreaterThan)(nil) // GreaterThan corresponds to gt(). type GreaterThan struct { From linkedql.PathStep `json:"from"` Value quad.Value `json:"value"` } // Description implements Step. func (s *GreaterThan) Description() string { return "Greater than equals filters out values that are not greater than given value" } // BuildPath implements linkedql.PathStep. func (s *GreaterThan) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Filter(iterator.CompareGT, linkedql.AbsoluteValue(s.Value, ns)), nil } ================================================ FILE: query/linkedql/steps/greater_than_equals.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&GreaterThanEquals{}) } var _ linkedql.PathStep = (*GreaterThanEquals)(nil) // GreaterThanEquals corresponds to gte(). type GreaterThanEquals struct { From linkedql.PathStep `json:"from"` Value quad.Value `json:"value"` } // Description implements Step. func (s *GreaterThanEquals) Description() string { return "Greater than equals filters out values that are not greater than or equal given value" } // BuildPath implements linkedql.PathStep. func (s *GreaterThanEquals) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Filter(iterator.CompareGTE, linkedql.AbsoluteValue(s.Value, ns)), nil } ================================================ FILE: query/linkedql/steps/has.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Has{}) } var _ linkedql.PathStep = (*Has)(nil) // Has corresponds to .has(). type Has struct { From linkedql.PathStep `json:"from"` Property *linkedql.PropertyPath `json:"property"` Values []quad.Value `json:"values"` } // Description implements Step. func (s *Has) Description() string { return "filters all paths which are, at this point, on the subject for the given predicate and object, but do not follow the path, merely filter the possible paths. Usually useful for starting with all nodes, or limiting to a subset depending on some predicate/value pair." } // BuildPath implements linkedql.PathStep. func (s *Has) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } viaPath, err := s.Property.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Has(viaPath, linkedql.AbsoluteValues(s.Values, ns)...), nil } ================================================ FILE: query/linkedql/steps/has_reverse.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&HasReverse{}) } var _ linkedql.PathStep = (*HasReverse)(nil) // HasReverse corresponds to .hasR(). type HasReverse struct { From linkedql.PathStep `json:"from"` Property *linkedql.PropertyPath `json:"property"` Values []quad.Value `json:"values"` } // Description implements Step. func (s *HasReverse) Description() string { return "is the same as Has, but sets constraint in reverse direction." } // BuildPath implements linkedql.PathStep. func (s *HasReverse) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } viaPath, err := s.Property.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.HasReverse(viaPath, linkedql.AbsoluteValues(s.Values, ns)...), nil } ================================================ FILE: query/linkedql/steps/in.go ================================================ package steps import ( "github.com/cayleygraph/cayley/query/linkedql" ) func init() { linkedql.Register(&In{}) } var _ linkedql.PathStep = (*In)(nil) // In is an alias for ViewReverse. type In struct { VisitReverse } // Description implements Step. func (s *In) Description() string { return "aliases for ViewReverse" } ================================================ FILE: query/linkedql/steps/intersect.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Intersect{}) } var _ linkedql.PathStep = (*Intersect)(nil) // Intersect represents .intersect() and .and(). type Intersect struct { From linkedql.PathStep `json:"from"` Steps []linkedql.PathStep `json:"steps"` } // Description implements Step. func (s *Intersect) Description() string { return "resolves to all the same values resolved by the from step and the provided steps." } // BuildPath implements linkedql.PathStep. func (s *Intersect) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } p := fromPath for _, step := range s.Steps { stepPath, err := step.BuildPath(qs, ns) if err != nil { return nil, err } p = p.And(stepPath) } return p, nil } ================================================ FILE: query/linkedql/steps/jsonld_util.go ================================================ package steps import ( "fmt" ) type ldArray = []interface{} type ldMap = map[string]interface{} func unwrapValue(i interface{}) interface{} { m, ok := i.(ldMap) if ok && len(m) == 1 { v, ok := m["@value"] if ok { return v } } return i } func unwrapSingle(i interface{}) interface{} { a, ok := i.(ldArray) if ok && len(a) == 1 { return a[0] } return i } // isomorphic checks whether source and target JSON-LD structures are the same // semantically. This function is not complete and is maintained for testing // purposes. Hopefully in the future it can be proven sufficient for general // purpose use. func isomorphic(source interface{}, target interface{}) error { source = unwrapValue(unwrapSingle(source)) target = unwrapValue(unwrapSingle(target)) switch s := source.(type) { case string: t, ok := target.(string) if !ok { return fmt.Errorf("Expected %v to be a string but instead received %T", target, target) } if s != t { return fmt.Errorf("Expected \"%v\" but instead received \"%v\"", s, t) } return nil case ldArray: t, ok := target.(ldArray) if !ok { return fmt.Errorf("Expected multiple values but instead received the single value: %#v", target) } if len(s) != len(t) { return fmt.Errorf("Expected %#v and %#v to have the same length", s, t) } items: for _, i := range s { for _, tI := range t { if isomorphic(i, tI) == nil { continue items } } return fmt.Errorf("No matching values for the item %#v in %#v", i, t) } return nil case ldMap: t, ok := target.(ldMap) if !ok { return fmt.Errorf("Expected %#v to be a map or a slice with a single map but instead received %T", target, target) } for k, v := range s { tV, _ := t[k] err := isomorphic(v, tV) if err != nil { return err } } } return nil } ================================================ FILE: query/linkedql/steps/jsonld_util_test.go ================================================ package steps import ( "fmt" "testing" "github.com/stretchr/testify/require" ) var testCases = []struct { name string source interface{} target interface{} expected error }{ { name: "Single matching IDs", source: map[string]interface{}{"@id": "a"}, target: map[string]interface{}{"@id": "a"}, expected: nil, }, { name: "Single non matching IDs", source: map[string]interface{}{"@id": "a"}, target: map[string]interface{}{"@id": "b"}, expected: fmt.Errorf("Expected \"a\" but instead received \"b\""), }, { name: "Single matching properties", source: map[string]interface{}{"http://example.com/name": "Alice"}, target: map[string]interface{}{"http://example.com/name": "Alice"}, expected: nil, }, { name: "Single non matching properties", source: map[string]interface{}{"http://example.com/name": "Alice"}, target: map[string]interface{}{"http://example.com/name": "Bob"}, expected: fmt.Errorf("Expected \"Alice\" but instead received \"Bob\""), }, { name: "Single matching property with multiple values ordered", source: map[string]interface{}{"http://example.com/name": []interface{}{"Alice", "Bob"}}, target: map[string]interface{}{"http://example.com/name": []interface{}{"Alice", "Bob"}}, expected: nil, }, { name: "Single matching property with multiple values unordered", source: map[string]interface{}{"http://example.com/name": []interface{}{"Alice", "Bob"}}, target: map[string]interface{}{"http://example.com/name": []interface{}{"Bob", "Alice"}}, expected: nil, }, { name: "Single non matching property with multiple values", source: map[string]interface{}{"http://example.com/name": []interface{}{"Alice", "Bob"}}, target: map[string]interface{}{"http://example.com/name": []interface{}{"Dan", "Alice"}}, expected: fmt.Errorf("No matching values for the item \"Bob\" in []interface {}{\"Dan\", \"Alice\"}"), }, { name: "Single non matching property with multiple values non matching length", source: map[string]interface{}{"http://example.com/name": []interface{}{"Alice", "Bob"}}, target: map[string]interface{}{"http://example.com/name": []interface{}{"Alice"}}, expected: fmt.Errorf("Expected multiple values but instead received the single value: \"Alice\""), }, { name: "Single matching nested", source: map[string]interface{}{ "http://example.com/friend": map[string]interface{}{ "@id": "alice", }, }, target: map[string]interface{}{ "http://example.com/friend": map[string]interface{}{ "@id": "alice", }, }, expected: nil, }, { name: "Single non matching nested", source: map[string]interface{}{ "http://example.com/friend": map[string]interface{}{ "@id": "alice", }, }, target: map[string]interface{}{ "http://example.com/friend": map[string]interface{}{ "@id": "bob", }, }, expected: fmt.Errorf("Expected \"alice\" but instead received \"bob\""), }, { name: "Single matching properties with @value string", source: map[string]interface{}{"http://example.com/name": map[string]interface{}{"@value": "Alice"}}, target: map[string]interface{}{"http://example.com/name": map[string]interface{}{"@value": "Alice"}}, expected: nil, }, { name: "Single non matching properties with @value string", source: map[string]interface{}{"http://example.com/name": map[string]interface{}{"@value": "Alice"}}, target: map[string]interface{}{"http://example.com/name": map[string]interface{}{"@value": "Bob"}}, expected: fmt.Errorf("Expected \"Alice\" but instead received \"Bob\""), }, { name: "Single matching properties with @value string and string", source: map[string]interface{}{"http://example.com/name": map[string]interface{}{"@value": "Alice"}}, target: map[string]interface{}{"http://example.com/name": "Alice"}, expected: nil, }, { name: "Single matching properties with string and @value string", source: map[string]interface{}{"http://example.com/name": "Alice"}, target: map[string]interface{}{"http://example.com/name": map[string]interface{}{"@value": "Alice"}}, expected: nil, }, { name: "Single matching properties with @value string array string", source: map[string]interface{}{"http://example.com/name": []interface{}{map[string]interface{}{"@value": "Alice"}}}, target: map[string]interface{}{"http://example.com/name": "Alice"}, expected: nil, }, { name: "Single matching properties with string and @value string array", source: map[string]interface{}{"http://example.com/name": "Alice"}, target: map[string]interface{}{"http://example.com/name": []interface{}{map[string]interface{}{"@value": "Alice"}}}, expected: nil, }, } func TestIsomorphic(t *testing.T) { for _, c := range testCases { t.Run(c.name, func(t *testing.T) { require.Equal(t, c.expected, isomorphic(c.source, c.target)) }) } } ================================================ FILE: query/linkedql/steps/labels.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Labels{}) } var _ linkedql.PathStep = (*Labels)(nil) // Labels corresponds to .labels(). type Labels struct { From linkedql.PathStep `json:"from"` } // Description implements Step. func (s *Labels) Description() string { return "gets the list of inbound and outbound quad labels" } // BuildPath implements linkedql.PathStep. func (s *Labels) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Labels(), nil } ================================================ FILE: query/linkedql/steps/less_than.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&LessThan{}) } var _ linkedql.PathStep = (*LessThan)(nil) // LessThan corresponds to lt(). type LessThan struct { From linkedql.PathStep `json:"from"` Value quad.Value `json:"value"` } // Description implements Step. func (s *LessThan) Description() string { return "Less than filters out values that are not less than given value" } // BuildPath implements linkedql.PathStep. func (s *LessThan) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Filter(iterator.CompareLT, s.Value), nil } ================================================ FILE: query/linkedql/steps/less_than_equals.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&LessThanEquals{}) } var _ linkedql.PathStep = (*LessThanEquals)(nil) // LessThanEquals corresponds to lte(). type LessThanEquals struct { From linkedql.PathStep `json:"from"` Value quad.Value `json:"value"` } // Description implements Step. func (s *LessThanEquals) Description() string { return "Less than equals filters out values that are not less than or equal given value" } // BuildPath implements linkedql.PathStep. func (s *LessThanEquals) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Filter(iterator.CompareLTE, linkedql.AbsoluteValue(s.Value, ns)), nil } ================================================ FILE: query/linkedql/steps/like.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Like{}) } var _ linkedql.PathStep = (*Like)(nil) // Like corresponds to like(). type Like struct { From linkedql.PathStep `json:"from"` Pattern string `json:"likePattern"` } // Description implements Operator. func (s *Like) Description() string { return "Like filters out values that do not match given pattern." } // BuildPath implements PathStep. func (s *Like) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Filters(shape.Wildcard{Pattern: s.Pattern}), nil } ================================================ FILE: query/linkedql/steps/limit.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Limit{}) } var _ linkedql.PathStep = (*Limit)(nil) // Limit corresponds to .limit(). type Limit struct { From linkedql.PathStep `json:"from"` Limit int64 `json:"limit"` } // Description implements Step. func (s *Limit) Description() string { return "limits a number of nodes for current path." } // BuildPath implements linkedql.PathStep. func (s *Limit) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Limit(s.Limit), nil } ================================================ FILE: query/linkedql/steps/match.go ================================================ package steps import ( "fmt" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/jsonld" "github.com/cayleygraph/quad/voc" "github.com/cayleygraph/quad/voc/rdf" "github.com/cayleygraph/quad/voc/rdfs" ) func init() { linkedql.Register(&Match{}) } var _ linkedql.PathStep = (*Match)(nil) // Match corresponds to .has(). type Match struct { From linkedql.PathStep `json:"from" minCardinality:"0"` Pattern linkedql.GraphPattern `json:"pattern"` } // Description implements Step. func (s *Match) Description() string { return "filters all paths which are, at this point, on the subject for the given predicate and object, but do not follow the path, merely filter the possible paths. Usually useful for starting with all nodes, or limiting to a subset depending on some predicate/value pair." } // BuildPath implements linkedql.PathStep. func (s *Match) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { var p *path.Path if s.From != nil { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } p = fromPath } else { p = path.StartPath(qs) } // Get quads quads, err := parsePattern(s.Pattern, ns) if err != nil { return nil, err } return p.Follow(buildPatternPath(quads, ns)), nil } type entityProperties = map[quad.Value]map[quad.Value][]quad.Value func entityPropertiesToPath(entity quad.Value, entities entityProperties) *path.Path { p := path.StartMorphism() if iri, ok := entity.(quad.IRI); ok { p = p.Is(iri) } for property, values := range entities[entity] { for _, value := range values { if _, ok := value.(quad.BNode); ok { p = p.Out(property).Follow(entityPropertiesToPath(value, entities)).Back("") } else { p = p.Has(property, value) } } } return p } func groupEntities(pattern []quad.Quad, ns *voc.Namespaces) (entityProperties, map[quad.Value]struct{}) { // referenced holds entities used as values for properties of other entities referenced := make(map[quad.Value]struct{}) entities := make(entityProperties) for _, q := range pattern { entity := linkedql.AbsoluteValue(q.Subject, ns) property := linkedql.AbsoluteValue(q.Predicate, ns) value := linkedql.AbsoluteValue(q.Object, ns) if _, ok := value.(quad.BNode); ok { referenced[value] = struct{}{} } properties, ok := entities[entity] if !ok { properties = make(map[quad.Value][]quad.Value) entities[entity] = properties } if isSingleEntityQuad(q) { continue } properties[property] = append(properties[property], value) } return entities, referenced } // buildPath for given pattern constructs a Path object func buildPatternPath(pattern []quad.Quad, ns *voc.Namespaces) *path.Path { entities, referenced := groupEntities(pattern, ns) p := path.StartMorphism() for entity := range entities { // Apply only on entities which are not referenced as values if _, ok := referenced[entity]; !ok { p = p.Follow(entityPropertiesToPath(entity, entities)) } } return p } func contextualizePattern(pattern linkedql.GraphPattern, ns *voc.Namespaces) linkedql.GraphPattern { context := make(map[string]interface{}) for _, namespace := range ns.List() { context[namespace.Prefix] = namespace.Full } patternClone := linkedql.GraphPattern{ "@context": context, } for key, value := range pattern { patternClone[key] = value } return pattern } func quadsFromMap(o interface{}) ([]quad.Quad, error) { reader := jsonld.NewReaderFromMap(o) return quad.ReadAll(reader) } func normalizeQuads(quads []quad.Quad, pattern linkedql.GraphPattern) ([]quad.Quad, error) { if id, ok := pattern["@id"]; ok && len(quads) == 0 { idString, ok := id.(string) if !ok { return nil, fmt.Errorf("Unexpected type for @id %T", idString) } quads = append(quads, makeSingleEntityQuad(quad.IRI(idString))) } return quads, nil } func parsePattern(pattern linkedql.GraphPattern, ns *voc.Namespaces) ([]quad.Quad, error) { contextualizedPattern := contextualizePattern(pattern, ns) quads, err := quadsFromMap(contextualizedPattern) if err != nil { return nil, err } quads, err = normalizeQuads(quads, contextualizedPattern) if err != nil { return nil, err } if len(quads) == 0 && len(pattern) != 0 { return nil, fmt.Errorf("Pattern does not parse to any quad. `{}` is the only pattern allowed to not parse to any quad") } return quads, nil } // makeSingleEntityQuad creates a quad representing a propertyless entity. The // quad declares the entity is of type Resource, the base type of all entities // in RDF. func makeSingleEntityQuad(id quad.IRI) quad.Quad { return quad.Quad{Subject: id, Predicate: quad.IRI(rdf.Type), Object: quad.IRI(rdfs.Resource)} } func isSingleEntityQuad(q quad.Quad) bool { // rdf:type rdfs:Resource is always true but not expressed in the graph. // it is used to specify an entity without specifying a property. return q.Predicate == quad.IRI(rdf.Type) && q.Object == quad.IRI(rdfs.Resource) } ================================================ FILE: query/linkedql/steps/match_test.go ================================================ package steps import ( "testing" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" "github.com/stretchr/testify/require" ) var ( ns = "http://example.com/" alice = quad.IRI(ns + "alice") likes = quad.IRI(ns + "likes") name = quad.IRI(ns + "name") bob = quad.IRI(ns + "bob") address = quad.IRI(ns + "address") city = quad.IRI(ns + "city") street = quad.IRI(ns + "street") country = quad.IRI(ns + "country") ) var patternTestCases = []struct { name string pattern map[string]interface{} expected *path.Path }{ { name: "Empty Pattern", pattern: map[string]interface{}{}, expected: path.StartMorphism(), }, { name: "Single Entity", pattern: map[string]interface{}{ "@id": string(alice), }, expected: path.StartMorphism().Is(alice), }, { name: "Single Property Value", pattern: map[string]interface{}{ string(likes): map[string]interface{}{"@id": string(bob)}, }, expected: path.StartMorphism().Has(likes, bob), }, { name: "Nested Structure", pattern: map[string]interface{}{ string(address): map[string]interface{}{ string(street): "Lafayette", }, }, expected: path. StartMorphism(). Out(address). Follow( path.StartMorphism(). Has(street, quad.String("Lafayette")), ). Back(""), }, { name: "Two Level Nested Structure", pattern: map[string]interface{}{ string(address): map[string]interface{}{ string(country): map[string]interface{}{ string(name): "The United States of America", }, }, }, expected: path. StartMorphism(). Out(address). Follow( path.StartMorphism(). Out(country). Follow( path.StartMorphism(). Has(name, quad.String("The United States of America")), ). Back(""), ). Back(""), }, } func TestBuildPath(t *testing.T) { for _, c := range patternTestCases { t.Run(c.name, func(t *testing.T) { ns := voc.Namespaces{} quads, err := parsePattern(c.pattern, &ns) require.NoError(t, err) p := buildPatternPath(quads, &ns) expectedShape := c.expected.Shape() shape := p.Shape() // TODO(iddan): replace with stable comparison. Currently, it breaks // because order of properties is not guaranteed require.Equal(t, expectedShape, shape) }) } } ================================================ FILE: query/linkedql/steps/optional.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Optional{}) } var _ linkedql.PathStep = (*Optional)(nil) // Optional corresponds to .optional(). type Optional struct { From linkedql.PathStep `json:"from"` Step linkedql.PathStep `json:"step"` } // Description implements Step. func (s *Optional) Description() string { return "attempts to follow the given path from the current entity / value, if fails the entity / value will still be kept in the results" } // BuildPath implements linkedql.PathStep. func (s *Optional) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } p, err := s.Step.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Optional(p), nil } ================================================ FILE: query/linkedql/steps/order.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Order{}) } var _ linkedql.PathStep = (*Order)(nil) // Order corresponds to .order(). type Order struct { From linkedql.PathStep `json:"from"` } // Description implements Step. func (s *Order) Description() string { return "sorts the results in ascending order according to the current entity / value" } // BuildPath implements linkedql.PathStep. func (s *Order) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Order(), nil } ================================================ FILE: query/linkedql/steps/out.go ================================================ package steps import ( "github.com/cayleygraph/cayley/query/linkedql" ) func init() { linkedql.Register(&Out{}) } var _ linkedql.PathStep = (*Out)(nil) // Out is an alias for View. type Out struct { Visit } // Description implements Step. func (s *Out) Description() string { return "aliases for View" } ================================================ FILE: query/linkedql/steps/placeholder.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Placeholder{}) } var _ linkedql.PathStep = (*Placeholder)(nil) // Placeholder corresponds to .Placeholder(). type Placeholder struct{} // Description implements Step. func (s *Placeholder) Description() string { return "is like Vertex but resolves to the values in the context it is placed in. It should only be used where a linkedql.PathStep is expected and can't be resolved on its own." } // BuildPath implements linkedql.PathStep. func (s *Placeholder) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { return path.StartMorphism(), nil } ================================================ FILE: query/linkedql/steps/properties.go ================================================ package steps import ( "fmt" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Properties{}) } var _ linkedql.PathStep = (*Properties)(nil) // Properties corresponds to .properties(). type Properties struct { From linkedql.PathStep `json:"from"` Names *linkedql.PropertyPath `json:"names"` } // Description implements Step. func (s *Properties) Description() string { return "adds tags for all properties of the current entity" } func resolveNames(names *linkedql.PropertyPath) (linkedql.PropertyIRIs, error) { if names == nil { return nil, fmt.Errorf("Not implemented: should tag all properties") } switch n := names.PropertyPathI.(type) { case linkedql.PropertyStep: return nil, fmt.Errorf("Not implemented: should use step to resolve to properties") case linkedql.PropertyIRIs: return n, nil case linkedql.PropertyIRIStrings: return n.PropertyIRIs(), nil case linkedql.PropertyIRI: return linkedql.PropertyIRIs{n}, nil case linkedql.PropertyIRIString: return linkedql.PropertyIRIs{linkedql.PropertyIRI(n)}, nil default: return nil, fmt.Errorf("Unexpected type") } } // BuildPath implements linkedql.PathStep. func (s *Properties) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } p := fromPath names, err := resolveNames(s.Names) if err != nil { return nil, err } for _, n := range names { name := quad.IRI(n).FullWith(ns) tag := string(name) p = p.Save(name, tag) } return p, nil } ================================================ FILE: query/linkedql/steps/property_names.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&PropertyNames{}) } var _ linkedql.PathStep = (*PropertyNames)(nil) // PropertyNames corresponds to .propertyNames(). type PropertyNames struct { From linkedql.PathStep `json:"from"` } // Description implements Step. func (s *PropertyNames) Description() string { return "gets the list of predicates that are pointing out from a node." } // BuildPath implements linkedql.PathStep. func (s *PropertyNames) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.OutPredicates(), nil } ================================================ FILE: query/linkedql/steps/property_names_as.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&PropertyNamesAs{}) } var _ linkedql.PathStep = (*PropertyNamesAs)(nil) // PropertyNamesAs corresponds to .propertyNamesAs(). type PropertyNamesAs struct { From linkedql.PathStep `json:"from"` Tag string `json:"tag"` } // Description implements Step. func (s *PropertyNamesAs) Description() string { return "tags the list of predicates that are pointing out from a node." } // BuildPath implements linkedql.PathStep. func (s *PropertyNamesAs) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.SavePredicates(false, s.Tag), nil } ================================================ FILE: query/linkedql/steps/regexp.go ================================================ package steps import ( "regexp" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&RegExp{}) } var _ linkedql.PathStep = (*RegExp)(nil) // RegExp corresponds to regex(). type RegExp struct { From linkedql.PathStep `json:"from"` Expression string `json:"expression"` IncludeIRIs bool `json:"includeIRIs,omitempty"` } // Description implements Step. func (s *RegExp) Description() string { return "RegExp filters out values that do not match given pattern. If includeIRIs is set to true it matches IRIs in addition to literals." } // BuildPath implements PathStep. func (s *RegExp) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } pattern, err := regexp.Compile(s.Expression) if err != nil { return nil, err } if s.IncludeIRIs { return fromPath.RegexWithRefs(pattern), nil } return fromPath.RegexWithRefs(pattern), nil } ================================================ FILE: query/linkedql/steps/reverse_properties.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&ReverseProperties{}) } var _ linkedql.PathStep = (*ReverseProperties)(nil) // ReverseProperties corresponds to .reverseProperties(). type ReverseProperties struct { From linkedql.PathStep `json:"from"` Names *linkedql.PropertyPath `json:"names"` } // Description implements Step. func (s *ReverseProperties) Description() string { return "gets all the properties the current entity / value is referenced at" } // BuildPath implements linkedql.PathStep. func (s *ReverseProperties) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } p := fromPath names, err := resolveNames(s.Names) if err != nil { return nil, err } for _, n := range names { name := quad.IRI(n).FullWith(ns) tag := string(name) p = fromPath.SaveReverse(name, tag) } return p, nil } ================================================ FILE: query/linkedql/steps/reverse_property_names.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&ReversePropertyNames{}) } var _ linkedql.PathStep = (*ReversePropertyNames)(nil) // ReversePropertyNames corresponds to .reversePropertyNames(). type ReversePropertyNames struct { From linkedql.PathStep `json:"from"` } // Description implements Step. func (s *ReversePropertyNames) Description() string { return "gets the list of predicates that are pointing in to a node." } // BuildPath implements linkedql.PathStep. func (s *ReversePropertyNames) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.InPredicates(), nil } ================================================ FILE: query/linkedql/steps/reverse_property_names_as.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&ReversePropertyNamesAs{}) } var _ linkedql.PathStep = (*ReversePropertyNamesAs)(nil) // ReversePropertyNamesAs corresponds to .reversePropertyNamesAs(). type ReversePropertyNamesAs struct { From linkedql.PathStep `json:"from"` Tag string `json:"tag"` } // Description implements Step. func (s *ReversePropertyNamesAs) Description() string { return "tags the list of predicates that are pointing in to a node." } // BuildPath implements linkedql.PathStep. func (s *ReversePropertyNamesAs) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.SavePredicates(true, s.Tag), nil } ================================================ FILE: query/linkedql/steps/skip.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Skip{}) } var _ linkedql.PathStep = (*Skip)(nil) // Skip corresponds to .skip(). type Skip struct { From linkedql.PathStep `json:"from"` Offset int64 `json:"offset"` } // Description implements Step. func (s *Skip) Description() string { return "skips a number of nodes for current path." } // BuildPath implements linkedql.PathStep. func (s *Skip) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Skip(s.Offset), nil } ================================================ FILE: query/linkedql/steps/steps_final.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Select{}) linkedql.Register(&Documents{}) } var _ linkedql.IteratorStep = (*Select)(nil) // Select corresponds to .select(). type Select struct { Properties []string `json:"properties"` From linkedql.PathStep `json:"from"` ExcludeID bool `json:"excludeID"` } // Description implements Step. func (s *Select) Description() string { return "Select returns flat records of tags matched in the query" } // BuildIterator implements IteratorStep func (s *Select) BuildIterator(qs graph.QuadStore, ns *voc.Namespaces) (query.Iterator, error) { valueIt, err := linkedql.NewValueIteratorFromPathStep(s.From, qs, ns) if err != nil { return nil, err } it := linkedql.NewTagsIterator(valueIt, s.Properties, s.ExcludeID) return &it, nil } var _ linkedql.IteratorStep = (*Documents)(nil) // Documents corresponds to .documents(). type Documents struct { From linkedql.PathStep `json:"from"` } // Description implements Step. func (s *Documents) Description() string { return "Documents return documents of the tags matched in the query associated with their entity" } // BuildIterator implements IteratorStep func (s *Documents) BuildIterator(qs graph.QuadStore, ns *voc.Namespaces) (query.Iterator, error) { p, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } it, err := linkedql.NewValueIterator(p, qs), nil if err != nil { return nil, err } return linkedql.NewDocumentIterator(it), nil } ================================================ FILE: query/linkedql/steps/steps_test.go ================================================ package steps import ( "bytes" "context" "encoding/json" "fmt" "io/ioutil" "path/filepath" "strings" "testing" "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/jsonld" "github.com/cayleygraph/quad/voc" "github.com/stretchr/testify/require" ) type TestCase struct { Data interface{} `json:"data"` Query interface{} `json:"query"` Results interface{} `json:"results"` } func readData(data interface{}) ([]quad.Quad, error) { d, err := json.Marshal(data) if err != nil { return nil, err } b := bytes.NewBuffer(d) reader := jsonld.NewReader(b) quads, err := quad.ReadAll(reader) if err != nil { return nil, err } return quads, nil } func readQuery(raw interface{}) (linkedql.Step, error) { d, err := json.Marshal(raw) if err != nil { return nil, err } q, err := linkedql.Unmarshal(d) if err != nil { return nil, err } query, ok := q.(linkedql.Step) if !ok { return nil, fmt.Errorf("Expected linkedql.Step") } return query, nil } func TestLinkedQL(t *testing.T) { // Using files directory := "test-cases" files, err := ioutil.ReadDir(directory) if err != nil { require.NoError(t, err) } for _, info := range files { fileName := info.Name() filePath := filepath.Join(directory, fileName) if !strings.HasSuffix(fileName, ".json") { // skip non-JSON files continue } testName := strings.TrimSuffix(fileName, filepath.Ext(fileName)) t.Run(testName, func(t *testing.T) { file, err := ioutil.ReadFile(filePath) require.NoError(t, err) var c TestCase err = json.Unmarshal(file, &c) require.NoError(t, err) data, err := readData(c.Data) require.NoError(t, err, fileName) require.NotEmpty(t, data, fileName) query, err := readQuery(c.Query) require.NoError(t, err, fileName) require.NotNil(t, query, fileName) store := memstore.New(data...) voc := voc.Namespaces{} ctx := context.TODO() iterator, err := linkedql.BuildIterator(query, store, &voc) require.NoError(t, err) var results []interface{} for iterator.Next(ctx) { results = append(results, iterator.Result()) } require.NoError(t, iterator.Err()) require.Equal(t, nil, isomorphic(c.Results, results)) }) } } ================================================ FILE: query/linkedql/steps/test-cases/all-vertices.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Vertex", "values": [] }, "results": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/likes" }, { "@id": "http://example.com/bob" } ] } ================================================ FILE: query/linkedql/steps/test-cases/back.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Back", "from": { "@type": "Visit", "from": { "@type": "Match", "pattern": { "@id": "http://example.com/alice" } }, "properties": ["http://example.com/likes"] }, "name": "" }, "results": [{ "@id": "http://example.com/alice" }] } ================================================ FILE: query/linkedql/steps/test-cases/both.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@graph": [ { "@id": "bob", "likes": { "@id": "alice" } }, { "@id": "dan", "likes": { "@id": "bob" } } ] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Both", "from": { "@type": "Match", "pattern": { "@id": "http://example.com/bob" } }, "properties": "http://example.com/likes" }, "results": [ { "@id": "http://example.com/dan" }, { "@id": "http://example.com/alice" } ] } ================================================ FILE: query/linkedql/steps/test-cases/collect.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "friends": { "@list": [{ "@id": "alice" }, { "@id": "bob" }] } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Documents", "from": { "@type": "Collect", "from": { "@type": "Match", "pattern": {} }, "name": "http://example.com/friends" } }, "results": [ { "http://example.com/friends": { "@list": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/bob" } ] } } ] } ================================================ FILE: query/linkedql/steps/test-cases/count.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Count", "from": { "@type": "Match", "pattern": {} } }, "results": [4] } ================================================ FILE: query/linkedql/steps/test-cases/difference.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Difference", "from": { "@type": "Vertex", "values": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/likes" } ] }, "steps": [ { "@type": "Match", "pattern": { "@id": "http://example.com/likes" } } ] }, "results": [{ "@id": "http://example.com/alice" }] } ================================================ FILE: query/linkedql/steps/test-cases/documents.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@graph": [ { "@id": "alice", "likes": { "@id": "bob" }, "name": "Alice" }, { "@id": "bob", "likes": { "@id": "alice" }, "name": "Bob" } ] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Documents", "from": { "@type": "Properties", "from": { "@type": "Match", "pattern": {} }, "names": ["http://example.com/name", "http://example.com/likes"] } }, "results": { "@graph": [ { "@id": "http://example.com/alice", "http://example.com/likes": [{ "@id": "http://example.com/bob" }], "http://example.com/name": ["Alice"] }, { "@id": "http://example.com/bob", "http://example.com/likes": [{ "@id": "http://example.com/alice" }], "http://example.com/name": ["Bob"] } ] } } ================================================ FILE: query/linkedql/steps/test-cases/greater-than-equals.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "name": [0, 1, 2] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "GreaterThanEquals", "from": { "@type": "Match", "pattern": {} }, "value": 1 }, "results": [1, 2] } ================================================ FILE: query/linkedql/steps/test-cases/greater-than.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "name": [0, 1] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "GreaterThan", "from": { "@type": "Match", "pattern": {} }, "value": 0 }, "results": [1] } ================================================ FILE: query/linkedql/steps/test-cases/has-reverse.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "HasReverse", "from": { "@type": "Match", "pattern": {} }, "property": "http://example.com/likes", "values": [{ "@id": "http://example.com/alice" }] }, "results": [{ "@id": "http://example.com/bob" }] } ================================================ FILE: query/linkedql/steps/test-cases/has.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Has", "from": { "@type": "Match", "pattern": {} }, "property": "http://example.com/likes", "values": [{ "@id": "http://example.com/bob" }] }, "results": [{ "@id": "http://example.com/alice" }] } ================================================ FILE: query/linkedql/steps/test-cases/intersect.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@graph": [ { "@id": "bob", "likes": { "@id": "alice" } }, { "@id": "dan", "likes": { "@id": "alice" } } ] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Intersect", "from": { "@type": "Visit", "from": { "@type": "Match", "pattern": { "@id": "http://example.com/bob" } }, "properties": "http://example.com/likes" }, "steps": [ { "@type": "Visit", "from": { "@type": "Match", "values": { "@id": "http://example.com/bob" } }, "properties": "http://example.com/likes" } ] }, "results": [{ "@id": "http://example.com/alice" }] } ================================================ FILE: query/linkedql/steps/test-cases/less-than-equals.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "name": [-1, 0, 1] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "LessThanEquals", "from": { "@type": "Match", "pattern": {} }, "value": 0 }, "results": [-1, 0] } ================================================ FILE: query/linkedql/steps/test-cases/less-than.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "name": [0, 1] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "LessThan", "from": { "@type": "Match", "pattern": {} }, "value": 1 }, "results": [0] } ================================================ FILE: query/linkedql/steps/test-cases/like.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "name": "Alice" }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Like", "from": { "@type": "Match", "pattern": {} }, "likePattern": "%ali%" }, "results": [{ "@id": "http://example.com/alice" }] } ================================================ FILE: query/linkedql/steps/test-cases/limit.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Limit", "from": { "@type": "Match", "pattern": {} }, "limit": 2 }, "results": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/likes" } ] } ================================================ FILE: query/linkedql/steps/test-cases/match-all.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Match", "pattern": {} }, "results": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/likes" }, { "@id": "http://example.com/bob" } ] } ================================================ FILE: query/linkedql/steps/test-cases/match-exact.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Match", "pattern": { "@id": "http://example.com/alice" } }, "results": [{ "@id": "http://example.com/alice" }] } ================================================ FILE: query/linkedql/steps/test-cases/optional.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@graph": [ { "@id": "alice", "likes": { "@id": "bob" }, "name": "Alice" }, { "@id": "bob", "name": "Bob" } ] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "Optional", "from": { "@type": "Properties", "from": { "@type": "Match", "pattern": {} }, "names": ["http://example.com/name"] }, "step": { "@type": "Properties", "from": { "@type": "Placeholder" }, "names": ["http://example.com/likes"] } }, "tags": [] }, "results": [ { "http://example.com/likes": { "@id": "http://example.com/bob" }, "http://example.com/name": "Alice" }, { "http://example.com/name": "Bob" } ] } ================================================ FILE: query/linkedql/steps/test-cases/order.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Order", "from": { "@type": "Match", "pattern": {} } }, "results": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/bob" }, { "@id": "http://example.com/likes" } ] } ================================================ FILE: query/linkedql/steps/test-cases/properties.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "Properties", "from": { "@type": "Match", "pattern": {} }, "names": ["http://example.com/likes"] }, "tags": [] }, "results": [ { "http://example.com/likes": { "@id": "http://example.com/bob" } } ] } ================================================ FILE: query/linkedql/steps/test-cases/property-names-as.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "PropertyNamesAs", "from": { "@type": "Match", "pattern": {} }, "tag": "property" }, "tags": [] }, "results": [{ "property": { "@id": "http://example.com/likes" } }] } ================================================ FILE: query/linkedql/steps/test-cases/property-names.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "PropertyNames", "from": { "@type": "Match", "pattern": {} } }, "results": [{ "@id": "http://example.com/likes" }] } ================================================ FILE: query/linkedql/steps/test-cases/reg-exp.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "name": "Alice" }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "RegExp", "expression": "A", "from": { "@type": "Match", "pattern": {} }, "includeIRIs": false }, "results": ["Alice"] } ================================================ FILE: query/linkedql/steps/test-cases/reverse-properties.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "ReverseProperties", "from": { "@type": "Match", "pattern": {} }, "names": ["http://example.com/likes"] }, "tags": [] }, "results": [ { "http://example.com/likes": { "@id": "http://example.com/alice" } } ] } ================================================ FILE: query/linkedql/steps/test-cases/reverse-property-names-as.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "ReversePropertyNamesAs", "from": { "@type": "Match", "pattern": {} }, "tag": "property" }, "tags": [] }, "results": [{ "property": { "@id": "http://example.com/likes" } }] } ================================================ FILE: query/linkedql/steps/test-cases/select-with-tags.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "As", "from": { "@type": "Visit", "from": { "@type": "As", "from": { "@type": "Match", "pattern": {} }, "name": "http://example.com/liker" }, "properties": "http://example.com/likes" }, "name": "liked" }, "tags": ["http://example.com/liker"] }, "results": [ { "http://example.com/liker": { "@id": "http://example.com/alice" } } ] } ================================================ FILE: query/linkedql/steps/test-cases/select.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "Visit", "from": { "@type": "As", "from": { "@type": "Match", "pattern": {} }, "name": "http://example.com/liker" }, "properties": "http://example.com/likes" }, "tags": [] }, "results": [ { "http://example.com/liker": { "@id": "http://example.com/alice" } } ] } ================================================ FILE: query/linkedql/steps/test-cases/skip.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Skip", "from": { "@type": "Match", "pattern": {} }, "offset": 2 }, "results": [{ "@id": "http://example.com/bob" }] } ================================================ FILE: query/linkedql/steps/test-cases/union.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Union", "from": { "@type": "Match", "pattern": { "@id": "http://example.com/alice" } }, "steps": [ { "@type": "Match", "pattern": { "@id": "http://example.com/bob" } } ] }, "results": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/bob" } ] } ================================================ FILE: query/linkedql/steps/test-cases/unique.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Unique", "from": { "@type": "Vertex", "values": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/alice" }, { "@id": "http://example.com/bob" } ] } }, "results": [ { "@id": "http://example.com/alice" }, { "@id": "http://example.com/bob" } ] } ================================================ FILE: query/linkedql/steps/test-cases/view-reverse.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "VisitReverse", "from": { "@type": "Match", "pattern": {} }, "properties": "http://example.com/likes" }, "results": [{ "@id": "http://example.com/alice" }] } ================================================ FILE: query/linkedql/steps/test-cases/view.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@id": "alice", "likes": { "@id": "bob" } }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Visit", "from": { "@type": "Match", "pattern": {} }, "properties": "http://example.com/likes" }, "results": [{ "@id": "http://example.com/bob" }] } ================================================ FILE: query/linkedql/steps/test-cases/where.json ================================================ { "data": { "@context": { "@base": "http://example.com/", "@vocab": "http://example.com/" }, "@graph": [ { "@id": "alice", "likes": { "@id": "bob" }, "name": "Alice" }, { "@id": "bob", "name": "Bob" } ] }, "query": { "@context": { "@vocab": "http://cayley.io/linkedql#" }, "@type": "Select", "from": { "@type": "Properties", "from": { "@type": "Where", "condition": { "@type": "Has", "from": { "@type": "Placeholder" }, "property": "http://example.com/likes", "values": [{ "@id": "http://example.com/bob" }] }, "from": { "@type": "Match", "pattern": {} } }, "names": ["http://example.com/name"] }, "tags": [] }, "results": [{ "http://example.com/name": "Alice" }] } ================================================ FILE: query/linkedql/steps/union.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Union{}) } var _ linkedql.PathStep = (*Union)(nil) // Union corresponds to .union() and .or(). type Union struct { From linkedql.PathStep `json:"from"` Steps []linkedql.PathStep `json:"steps"` } // Description implements Step. func (s *Union) Description() string { return "returns the combined paths of the two queries. Notice that it's per-path, not per-node. Once again, if multiple paths reach the same destination, they might have had different ways of getting there (and different tags)." } // BuildPath implements linkedql.PathStep. func (s *Union) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } p := fromPath for _, step := range s.Steps { valuePath, err := step.BuildPath(qs, ns) if err != nil { return nil, err } p = p.Or(valuePath) } return p, nil } ================================================ FILE: query/linkedql/steps/unique.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Unique{}) } var _ linkedql.PathStep = (*Unique)(nil) // Unique corresponds to .unique(). type Unique struct { From linkedql.PathStep `json:"from"` } // Description implements Step. func (s *Unique) Description() string { return "removes duplicate values from the path." } // BuildPath implements linkedql.PathStep. func (s *Unique) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Unique(), nil } ================================================ FILE: query/linkedql/steps/vertex.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Vertex{}) } var _ linkedql.PathStep = (*Vertex)(nil) // Vertex corresponds to g.Vertex() and g.V(). type Vertex struct { Values []quad.Value `json:"values"` } // Description implements Step. func (s *Vertex) Description() string { return "resolves to all the existing objects and primitive values in the graph. If provided with values resolves to a sublist of all the existing values in the graph." } // BuildPath implements linkedql.PathStep. func (s *Vertex) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { return path.StartPath(qs, linkedql.AbsoluteValues(s.Values, ns)...), nil } ================================================ FILE: query/linkedql/steps/visit.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Visit{}) } var _ linkedql.PathStep = (*Visit)(nil) // Visit corresponds to .view(). type Visit struct { From linkedql.PathStep `json:"from"` Properties *linkedql.PropertyPath `json:"properties"` } // Description implements Step. func (s *Visit) Description() string { return "resolves to the values of the given property or properties in via of the current objects. If via is a path it's resolved values will be used as properties." } // BuildPath implements linkedql.PathStep. func (s *Visit) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } viaPath, err := s.Properties.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.Out(viaPath), nil } ================================================ FILE: query/linkedql/steps/visit_reverse.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&VisitReverse{}) } var _ linkedql.PathStep = (*VisitReverse)(nil) // VisitReverse corresponds to .viewReverse(). type VisitReverse struct { From linkedql.PathStep `json:"from"` Properties *linkedql.PropertyPath `json:"properties"` } // Description implements Step. func (s *VisitReverse) Description() string { return "is the inverse of View. Starting with the nodes in `path` on the object, follow the quads with predicates defined by `predicatePath` to their subjects." } // BuildPath implements linkedql.PathStep. func (s *VisitReverse) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } viaPath, err := s.Properties.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.In(viaPath), nil } ================================================ FILE: query/linkedql/steps/where.go ================================================ package steps import ( "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/linkedql" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad/voc" ) func init() { linkedql.Register(&Where{}) } var _ linkedql.PathStep = (*Where)(nil) // Where corresponds to .where(). type Where struct { From linkedql.PathStep `json:"from"` Condition linkedql.PathStep `json:"condition"` } // Description implements Step. func (s *Where) Description() string { return "filters results that fulfill a specified condition" } // BuildPath implements linkedql.PathStep. func (s *Where) BuildPath(qs graph.QuadStore, ns *voc.Namespaces) (*path.Path, error) { fromPath, err := s.From.BuildPath(qs, ns) if err != nil { return nil, err } stepPath, err := s.Condition.BuildPath(qs, ns) if err != nil { return nil, err } return fromPath.And(stepPath.Reverse()), nil } ================================================ FILE: query/linkedql/voc_util.go ================================================ package linkedql import ( "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) // AbsoluteValue uses given ns to resolve short IRIs and types in typed strings to their full form func AbsoluteValue(value quad.Value, ns *voc.Namespaces) quad.Value { switch v := value.(type) { case quad.IRI: return v.FullWith(ns) case quad.TypedString: return quad.TypedString{Value: v.Value, Type: v.Type.FullWith(ns)} default: return v } } // AbsoluteValues applies AbsoluteValue on each item in provided values using provided ns func AbsoluteValues(values []quad.Value, ns *voc.Namespaces) []quad.Value { var absoluteValues []quad.Value for _, value := range values { absoluteValues = append(absoluteValues, AbsoluteValue(value, ns)) } return absoluteValues } ================================================ FILE: query/mql/build_iterator.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mql import ( "context" "errors" "fmt" "math" "strings" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) func buildFixed(s string) shape.Shape { return shape.Lookup{quad.StringToValue(s)} } func buildAllResult(path Path) shape.Shape { return shape.Save{ From: shape.AllNodes{}, Tags: []string{string(path)}, } } func (q *Query) BuildIteratorTree(ctx context.Context, query interface{}) { q.isRepeated = make(map[Path]bool) q.queryStructure = make(map[Path]map[string]interface{}) q.queryResult = make(map[ResultPath]map[string]interface{}) q.queryResult[""] = make(map[string]interface{}) var ( opt bool s shape.Shape ) s, opt, q.err = q.buildShape(query, NewPath()) if q.err == nil && opt { q.err = errors.New("optional iterator at the top level") } q.it = shape.BuildIterator(ctx, q.ses.qs, s) } func (q *Query) buildShape(query interface{}, path Path) (s shape.Shape, optional bool, err error) { err = nil optional = false switch t := query.(type) { case bool: // for JSON booleans s = shape.Lookup{quad.Bool(t)} case float64: // for JSON numbers // Damn you, Javascript, and your lack of integer values. if math.Floor(t) == t { // Treat it like an integer. s = shape.Lookup{quad.Int(t)} } else { s = shape.Lookup{quad.Float(t)} } case string: // for JSON strings s = buildFixed(t) case []interface{}: // for JSON arrays q.isRepeated[path] = true if len(t) == 0 { s = buildAllResult(path) optional = true } else if len(t) == 1 { s, optional, err = q.buildShape(t[0], path) } else { err = fmt.Errorf("multiple fields at location root %s", path.DisplayString()) } case map[string]interface{}: // for JSON objects s, err = q.buildShapeMap(t, path) case nil: s = buildAllResult(path) optional = true default: err = fmt.Errorf("Unknown JSON type: %T", query) } if err != nil { return nil, false, err } s = shape.Save{ From: s, Tags: []string{string(path)}, } return s, optional, nil } func (q *Query) buildShapeMap(query map[string]interface{}, path Path) (shape.Shape, error) { it := shape.IntersectOpt{ Sub: shape.Intersect{ shape.AllNodes{}, }, } outputStructure := make(map[string]interface{}) for key, subquery := range query { optional := false outputStructure[key] = nil reverse := false pred := key if strings.HasPrefix(pred, "@") { i := strings.Index(pred, ":") if i != -1 { pred = pred[(i + 1):] } } if strings.HasPrefix(pred, "!") { reverse = true pred = strings.TrimPrefix(pred, "!") } // Other special constructs here var subit shape.Shape if key == "id" { var err error subit, optional, err = q.buildShape(subquery, path.Follow(key)) if err != nil { return nil, err } } else { var ( builtIt shape.Shape err error ) builtIt, optional, err = q.buildShape(subquery, path.Follow(key)) if err != nil { return nil, err } from, to := quad.Subject, quad.Object if reverse { from, to = to, from } subit = shape.NodesFrom{ Dir: from, Quads: shape.Quads{ {Dir: quad.Predicate, Values: buildFixed(pred)}, {Dir: to, Values: builtIt}, }, } } if optional { it.AddOptional(subit) } else { it.Add(subit) } } q.queryStructure[path] = outputStructure if len(it.Opt) == 0 { return it.Sub, nil } return it, nil } type byRecordLength []ResultPath func (p byRecordLength) Len() int { return len(p) } func (p byRecordLength) Less(i, j int) bool { iLen := len(strings.Split(string(p[i]), "\x30")) jLen := len(strings.Split(string(p[j]), "\x30")) if iLen < jLen { return true } if iLen == jLen { if len(string(p[i])) < len(string(p[j])) { return true } } return false } func (p byRecordLength) Swap(i, j int) { p[i], p[j] = p[j], p[i] } ================================================ FILE: query/mql/fill.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mql import ( "fmt" "sort" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/quad" ) func (q *Query) treeifyResult(tags map[string]graph.Ref) (map[ResultPath]string, error) { // Transform the map into something a little more interesting. results := make(map[Path]string) for k, v := range tags { if v == nil { continue } nv, err := q.ses.qs.NameOf(v) if err != nil { return nil, err } results[Path(k)] = quadValueToNative(nv) } resultPaths := make(map[ResultPath]string) for k, v := range results { resultPaths[k.ToResultPathFromMap(results)] = v } paths := make([]ResultPath, 0, len(resultPaths)) for path := range resultPaths { paths = append(paths, path) } sort.Sort(byRecordLength(paths)) // Build Structure for _, path := range paths { currentPath := path.getPath() value := resultPaths[path] namePath := path.AppendValue(value) if _, ok := q.queryResult[namePath]; !ok { targetPath, key := path.splitLastPath() if path == "" { targetPath, key = "", value if _, ok := q.queryResult[""][value]; !ok { q.resultOrder = append(q.resultOrder, value) } } if _, ok := q.queryStructure[currentPath]; ok { // If there's substructure, then copy that in. newStruct := q.copyPathStructure(currentPath) if q.isRepeated[currentPath] && currentPath != "" { switch t := q.queryResult[targetPath][key].(type) { case nil: x := make([]interface{}, 0) x = append(x, newStruct) q.queryResult[targetPath][key] = x q.queryResult[namePath] = newStruct case []interface{}: q.queryResult[targetPath][key] = append(t, newStruct) q.queryResult[namePath] = newStruct } } else { q.queryResult[namePath] = newStruct q.queryResult[targetPath][key] = newStruct } } } } // Fill values for _, path := range paths { currentPath := path.getPath() value, ok := resultPaths[path] if !ok { continue } namePath := path.AppendValue(value) if _, ok := q.queryStructure[currentPath]; ok { // We're dealing with ids. if _, ok := q.queryResult[namePath]["id"]; ok { q.queryResult[namePath]["id"] = value } } else { // Just a value. targetPath, key := path.splitLastPath() if q.isRepeated[currentPath] { switch t := q.queryResult[targetPath][key].(type) { case nil: x := make([]interface{}, 0) x = append(x, value) q.queryResult[targetPath][key] = x case []interface{}: q.queryResult[targetPath][key] = append(t, value) } } else { q.queryResult[targetPath][key] = value } } } return resultPaths, nil } func (q *Query) buildResults() { for _, v := range q.resultOrder { q.results = append(q.results, q.queryResult[""][v]) } } func quadValueToNative(v quad.Value) string { out := quad.NativeOf(v) if nv, ok := out.(quad.Value); ok && v == nv { return quad.StringOf(v) } return fmt.Sprint(out) } ================================================ FILE: query/mql/mql_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mql import ( "context" "encoding/json" "testing" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest/testutil" _ "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/query" _ "github.com/cayleygraph/cayley/writer" "github.com/cayleygraph/quad" ) // This is a simple test graph. // // +---+ +---+ // | A |------- ->| F |<-- // +---+ \------>+---+-/ +---+ \--+---+ // ------>|#B#| | | E | // +---+-------/ >+---+ | +---+ // | C | / v // +---+ -/ +---+ // ---- +---+/ |#G#| // \-->|#D#|------------->+---+ // +---+ // func makeTestSession(data []quad.Quad) *Session { qs, _ := graph.NewQuadStore("memstore", "", nil) w, _ := graph.NewQuadWriter("single", qs, nil) for _, t := range data { w.AddQuad(t) } return NewSession(qs) } var testQueries = []struct { message string query string tag string expect string }{ { message: "get all IDs in the database", query: `[{"id": null}]`, expect: ` [ {"id": ""}, {"id": ""}, {"id": ""}, {"id": ""}, {"id": ""}, {"id": "cool_person"}, {"id": ""}, {"id": ""}, {"id": ""}, {"id": ""}, {"id": ""}, {"id": ""}, {"id": "smart_person"}, {"id": ""} ] `, }, { message: "get nodes by status", query: `[{"id": null, "": "cool_person"}]`, expect: ` [ {"id": "", "": "cool_person"}, {"id": "", "": "cool_person"}, {"id": "", "": "cool_person"} ] `, }, { message: "show correct null semantics", query: `[{"id": "cool_person", "status": null}]`, expect: ` [ {"id": "cool_person", "status": null} ] `, }, { message: "get correct follows list", query: `[{"id": "", "": []}]`, expect: ` [ {"id": "", "": ["", ""]} ] `, }, { message: "get correct reverse follows list", query: `[{"id": "", "!": []}]`, expect: ` [ {"id": "", "!": ["", ""]} ] `, }, { message: "get correct follows struct", query: `[{"id": null, "": {"id": null, "": "cool_person"}}]`, expect: ` [ {"id": "", "": {"id": "", "": "cool_person"}}, {"id": "", "": {"id": "", "": "cool_person"}}, {"id": "", "": {"id": "", "": "cool_person"}}, {"id": "", "": {"id": "", "": "cool_person"}} ] `, }, { message: "get correct reverse follows struct", query: `[{"id": null, "!": [{"id": null, "" : "cool_person"}]}]`, expect: ` [ {"id": "", "!": [{"id": "", "": "cool_person"}]}, {"id": "", "!": [{"id": "", "": "cool_person"}]}, {"id": "", "!": [{"id": "", "": "cool_person"}]} ] `, }, { message: "get correct co-follows", query: `[{"id": null, "@A:": "", "@B:": ""}]`, expect: ` [ {"id": "", "@A:": "", "@B:": ""} ] `, }, { message: "get correct reverse co-follows", query: `[{"id": null, "!": {"id": ""}, "@A:!": ""}]`, expect: ` [ {"id": "", "!": {"id": ""}, "@A:!": ""} ] `, }, } func runQuery(t testing.TB, g []quad.Quad, qu string) interface{} { s := makeTestSession(g) ctx := context.TODO() it, err := s.Execute(ctx, qu, query.Options{Collation: query.JSON}) if err != nil { t.Fatal(err) } defer it.Close() var out []interface{} for it.Next(ctx) { out = append(out, it.Result()) } return out } func TestMQL(t *testing.T) { simpleGraph := testutil.LoadGraph(t, "../../data/testdata.nq") for _, test := range testQueries { t.Run(test.message, func(t *testing.T) { got := runQuery(t, simpleGraph, test.query) var expect interface{} json.Unmarshal([]byte(test.expect), &expect) require.Equal(t, expect, got) }) } } ================================================ FILE: query/mql/query.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mql import ( "fmt" "strings" "github.com/cayleygraph/cayley/graph/iterator" ) type ( Path string ResultPath string ) type Query struct { ses *Session it iterator.Shape isRepeated map[Path]bool queryStructure map[Path]map[string]interface{} queryResult map[ResultPath]map[string]interface{} results []interface{} resultOrder []string err error } func (q *Query) isError() bool { return q.err != nil } func (q *Query) copyPathStructure(path Path) map[string]interface{} { output := make(map[string]interface{}) for k, v := range q.queryStructure[path] { output[k] = v } return output } func NewPath() Path { return "" } func (p Path) Follow(s string) Path { return Path(fmt.Sprintf("%s\x1E%s", p, s)) } func (p Path) DisplayString() string { return strings.Replace(string(p), "\x1E", ".", -1) } func NewResultPath() ResultPath { return "" } func (p ResultPath) FollowPath(followPiece string, value string) ResultPath { if string(p) == "" { return ResultPath(fmt.Sprintf("%s\x1E%s", value, followPiece)) } return ResultPath(fmt.Sprintf("%s\x1E%s\x1E%s", p, value, followPiece)) } func (p ResultPath) getPath() Path { out := NewPath() pathPieces := strings.Split(string(p), "\x1E") for len(pathPieces) > 1 { a := pathPieces[1] pathPieces = pathPieces[2:] out = out.Follow(a) } return out } func (p ResultPath) splitLastPath() (ResultPath, string) { pathPieces := strings.Split(string(p), "\x1E") return ResultPath(strings.Join(pathPieces[:len(pathPieces)-1], "\x1E")), pathPieces[len(pathPieces)-1] } func (p ResultPath) AppendValue(value string) ResultPath { if string(p) == "" { return ResultPath(value) } return ResultPath(fmt.Sprintf("%s\x1E%s", p, value)) } func (p Path) ToResultPathFromMap(resultMap map[Path]string) ResultPath { output := NewResultPath() pathPieces := strings.Split(string(p), "\x1E")[1:] pathSoFar := NewPath() for _, piece := range pathPieces { output = output.FollowPath(piece, resultMap[pathSoFar]) pathSoFar = pathSoFar.Follow(piece) } return output } func NewQuery(ses *Session) *Query { var q Query q.ses = ses q.err = nil return &q } ================================================ FILE: query/mql/session.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mql import ( "context" "encoding/json" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query" ) const Name = "mql" func init() { query.RegisterLanguage(query.Language{ Name: Name, Session: func(qs graph.QuadStore) query.Session { return NewSession(qs) }, }) } type Session struct { qs graph.QuadStore } func NewSession(qs graph.QuadStore) *Session { return &Session{qs: qs} } type mqlIterator struct { q *Query col query.Collation it iterator.Scanner res []interface{} } func (it *mqlIterator) Next(ctx context.Context) bool { // TODO: stream results if it.res != nil { if len(it.res) == 0 { return false } it.res = it.res[1:] return len(it.res) != 0 } for it.it.Next(ctx) { m := make(map[string]graph.Ref) it.it.TagResults(m) it.q.treeifyResult(m) for it.it.NextPath(ctx) { m = make(map[string]graph.Ref, len(m)) it.it.TagResults(m) it.q.treeifyResult(m) } } if err := it.it.Err(); err != nil { return false } it.q.buildResults() it.res = it.q.results return len(it.res) != 0 } func (it *mqlIterator) Result() interface{} { if len(it.res) == 0 { return nil } return it.res[0] } func (it *mqlIterator) Err() error { return it.it.Err() } func (it *mqlIterator) Close() error { return it.it.Close() } func (s *Session) Execute(ctx context.Context, input string, opt query.Options) (query.Iterator, error) { switch opt.Collation { case query.REPL, query.JSON: default: return nil, &query.ErrUnsupportedCollation{Collation: opt.Collation} } var mqlQuery interface{} if err := json.Unmarshal([]byte(input), &mqlQuery); err != nil { return nil, err } q := NewQuery(s) q.BuildIteratorTree(ctx, mqlQuery) if q.isError() { return nil, q.err } it := q.it.Iterate() if opt.Limit > 0 { it = iterator.NewLimitNext(it, int64(opt.Limit)) } return &mqlIterator{ q: q, col: opt.Collation, it: it, }, nil } func (s *Session) Clear() { // Since we create a new Query underneath every query, clearing isn't necessary. return } ================================================ FILE: query/path/morphism_apply_functions.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package path import ( "context" "fmt" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) // join puts two iterators together by intersecting their result sets with an AND // Since we're using an and iterator, it's a good idea to put the smallest result // set first so that Next() produces fewer values to check Contains(). func join(its ...shape.Shape) shape.Shape { if len(its) == 0 { return shape.Null{} } else if _, ok := its[0].(shape.AllNodes); ok { return join(its[1:]...) } return shape.Intersect(its) } func joinOpt(main, opt shape.Shape) shape.Shape { return shape.IntersectOptional(main, opt) } // isMorphism represents all nodes passed in-- if there are none, this function // acts as a passthrough for the previous iterator. func isMorphism(nodes ...quad.Value) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return isMorphism(nodes...), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { if len(nodes) == 0 { // Acting as a passthrough here is equivalent to // building a NodesAllIterator to Next() or Contains() // from here as in previous versions. return in, ctx } s := shape.Lookup(nodes) if _, ok := in.(shape.AllNodes); ok { return s, ctx } // Anything with fixedIterators will usually have a much // smaller result set, so join isNodes first here. return join(s, in), ctx }, } } // isNodeMorphism represents all nodes passed in-- if there are none, this function // acts as a passthrough for the previous iterator. func isNodeMorphism(nodes ...graph.Ref) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return isNodeMorphism(nodes...), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { if len(nodes) == 0 { // Acting as a passthrough here is equivalent to // building a NodesAllIterator to Next() or Contains() // from here as in previous versions. return in, ctx } // Anything with fixedIterators will usually have a much // smaller result set, so join isNodes first here. return join(shape.Fixed(nodes), in), ctx }, } } // filterMorphism is the set of nodes that passes filters. func filterMorphism(filt []shape.ValueFilter) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return filterMorphism(filt), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.AddFilters(in, filt...), ctx }, } } // hasPathMorphism is a generic form of Has morphism - it accepts a subtree that will be checked on the current path. func hasPathMorphism(p *Path) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return hasPathMorphism(p), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.IntersectShapes(in, p.Shape()), ctx }, } } // hasMorphism is the set of nodes that is reachable via either a *Path, a // single node.(string) or a list of nodes.([]string). func hasMorphism(via interface{}, rev bool, nodes ...quad.Value) morphism { var node shape.Shape if len(nodes) == 0 { node = shape.AllNodes{} } else { node = shape.Lookup(nodes) } return hasShapeMorphism(via, rev, node) } // hasShapeMorphism is the set of nodes that is reachable via either a *Path, a // single node.(string) or a list of nodes.([]string). func hasShapeMorphism(via interface{}, rev bool, nodes shape.Shape) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return hasShapeMorphism(via, rev, nodes), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.HasLabels(in, buildVia(via), nodes, ctx.labelSet, rev), ctx }, } } // hasFilterMorphism is the set of nodes that is reachable via either a *Path, a // single node.(string) or a list of nodes.([]string) and that passes provided filters. func hasFilterMorphism(via interface{}, rev bool, filt []shape.ValueFilter) morphism { return hasShapeMorphism(via, rev, shape.Filter{ From: shape.AllNodes{}, Filters: filt, }) } func tagMorphism(tags ...string) morphism { return morphism{ IsTag: true, Reversal: func(ctx *pathContext) (morphism, *pathContext) { return tagMorphism(tags...), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Save{From: in, Tags: tags}, ctx }, tags: tags, } } // outMorphism iterates forward one RDF triple or via an entire path. func outMorphism(tags []string, via ...interface{}) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return inMorphism(tags, via...), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Out(in, buildVia(via...), ctx.labelSet, tags...), ctx }, tags: tags, } } // inMorphism iterates backwards one RDF triple or via an entire path. func inMorphism(tags []string, via ...interface{}) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return outMorphism(tags, via...), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.In(in, buildVia(via...), ctx.labelSet, tags...), ctx }, tags: tags, } } func bothMorphism(tags []string, via ...interface{}) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return bothMorphism(tags, via...), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { via := buildVia(via...) return shape.Union{ shape.In(in, via, ctx.labelSet, tags...), shape.Out(in, via, ctx.labelSet, tags...), }, ctx }, tags: tags, } } func labelContextMorphism(tags []string, via ...interface{}) morphism { var path shape.Shape if len(via) == 0 { path = nil } else { path = shape.Save{From: buildVia(via...), Tags: tags} } return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { out := ctx.copy() ctx.labelSet = path return labelContextMorphism(tags, via...), &out }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { out := ctx.copy() out.labelSet = path return in, &out }, tags: tags, } } // labelsMorphism iterates to the uniqified set of labels from // the given set of nodes in the path. func labelsMorphism() morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { panic("not implemented") }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Labels(in), ctx }, } } // predicatesMorphism iterates to the uniqified set of predicates from // the given set of nodes in the path. func predicatesMorphism(isIn bool) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { panic("not implemented: need a function from predicates to their associated edges") }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Predicates(in, isIn), ctx }, } } // savePredicatesMorphism tags either forward or reverse predicates from current node // without affecting path. func savePredicatesMorphism(isIn bool, tag string) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return savePredicatesMorphism(isIn, tag), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.SavePredicates(in, isIn, tag), ctx }, } } type iteratorShape struct { it iterator.Shape sent bool } func (s *iteratorShape) BuildIterator(qs graph.QuadStore) iterator.Shape { if s.sent { return iterator.NewError(fmt.Errorf("iterator already used in query")) } it := s.it s.it, s.sent = nil, true return it } func (s *iteratorShape) Optimize(ctx context.Context, r shape.Optimizer) (shape.Shape, bool) { return s, false } // iteratorMorphism simply tacks the input iterator onto the chain. func iteratorMorphism(it iterator.Shape) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return iteratorMorphism(it), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return join(&iteratorShape{it: it}, in), ctx }, } } // andMorphism sticks a path onto the current iterator chain. func andMorphism(p *Path) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return andMorphism(p), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return join(in, p.Shape()), ctx }, } } // andOptMorphism sticks a path onto the current iterator chain and makes it optional. func andOptMorphism(p *Path) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return andOptMorphism(p), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return joinOpt(in, p.Shape()), ctx }, } } // orMorphism is the union, vice intersection, of a path and the current iterator. func orMorphism(p *Path) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return orMorphism(p), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Union{in, p.Shape()}, ctx }, } } func followMorphism(p *Path) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return followMorphism(p.Reverse()), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return p.ShapeFrom(in), ctx }, } } type iteratorBuilder func(qs graph.QuadStore) iterator.Shape func (s iteratorBuilder) BuildIterator(qs graph.QuadStore) iterator.Shape { return s(qs) } func (s iteratorBuilder) Optimize(ctx context.Context, r shape.Optimizer) (shape.Shape, bool) { return s, false } func followRecursiveMorphism(p *Path, maxDepth int, depthTags []string) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return followRecursiveMorphism(p.Reverse(), maxDepth, depthTags), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return iteratorBuilder(func(qs graph.QuadStore) iterator.Shape { in := in.BuildIterator(qs) it := iterator.NewRecursive(in, p.MorphismFor(qs), maxDepth) for _, s := range depthTags { it.AddDepthTag(s) } return it }), ctx }, } } // exceptMorphism removes all results on p.(*Path) from the current iterators. func exceptMorphism(p *Path) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return exceptMorphism(p), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return join(in, shape.Except{From: shape.AllNodes{}, Exclude: p.Shape()}), ctx }, } } // uniqueMorphism removes duplicate values from current path. func uniqueMorphism() morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return uniqueMorphism(), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Unique{From: in}, ctx }, } } func saveMorphism(via interface{}, tag string) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return saveMorphism(via, tag), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.SaveViaLabels(in, buildVia(via), ctx.labelSet, tag, false, false), ctx }, tags: []string{tag}, } } func saveReverseMorphism(via interface{}, tag string) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return saveReverseMorphism(via, tag), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.SaveViaLabels(in, buildVia(via), ctx.labelSet, tag, true, false), ctx }, tags: []string{tag}, } } func saveOptionalMorphism(via interface{}, tag string) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return saveOptionalMorphism(via, tag), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.SaveViaLabels(in, buildVia(via), ctx.labelSet, tag, false, true), ctx }, tags: []string{tag}, } } func saveOptionalReverseMorphism(via interface{}, tag string) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return saveOptionalReverseMorphism(via, tag), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.SaveViaLabels(in, buildVia(via), ctx.labelSet, tag, true, true), ctx }, tags: []string{tag}, } } func buildVia(via ...interface{}) shape.Shape { if len(via) == 0 { return shape.AllNodes{} } else if len(via) == 1 { v := via[0] switch p := v.(type) { case nil: return shape.AllNodes{} case *Path: return p.Shape() case quad.Value: return shape.Lookup{p} case []quad.Value: return shape.Lookup(p) } } nodes := make([]quad.Value, 0, len(via)) for _, v := range via { qv, ok := quad.AsValue(v) if !ok { panic(fmt.Errorf("Invalid type passed to buildViaPath: %v (%T)", v, v)) } nodes = append(nodes, qv) } return shape.Lookup(nodes) } // skipMorphism will skip a number of values-- if there are none, this function // acts as a passthrough for the previous iterator. func skipMorphism(v int64) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return skipMorphism(v), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { if v == 0 { // Acting as a passthrough return in, ctx } return shape.Page{From: in, Skip: v}, ctx }, } } func orderMorphism() morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return orderMorphism(), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Sort{From: in}, ctx }, } } // limitMorphism will limit a number of values-- if number is negative or zero, this function // acts as a passthrough for the previous iterator. func limitMorphism(v int64) morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return limitMorphism(v), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { if v <= 0 { // Acting as a passthrough return in, ctx } return shape.Page{From: in, Limit: v}, ctx }, } } // countMorphism will return count of values. func countMorphism() morphism { return morphism{ Reversal: func(ctx *pathContext) (morphism, *pathContext) { return countMorphism(), ctx }, Apply: func(in shape.Shape, ctx *pathContext) (shape.Shape, *pathContext) { return shape.Count{Values: in}, ctx }, } } ================================================ FILE: query/path/path.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package path import ( "context" "regexp" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) type applyMorphism func(shape.Shape, *pathContext) (shape.Shape, *pathContext) type morphism struct { IsTag bool Reversal func(*pathContext) (morphism, *pathContext) Apply applyMorphism tags []string } // pathContext allows a high-level change to the way paths are constructed. Some // functions may change the context, causing following chained calls to act // differently. // // In a sense, this is a global state which can be changed as the path // continues. And as with dealing with any global state, care should be taken: // // When modifying the context in Apply(), please copy the passed struct, // modifying the relevant fields if need be (or pass the given context onward). // // Under Reversal(), any functions that wish to change the context should // appropriately change the passed context (that is, the context that came after // them will now be what the application of the function would have been) and // then yield a pointer to their own member context as the return value. // // For more examples, look at the morphisms which claim the individual fields. type pathContext struct { // TODO(dennwc): replace with net/context? // Represents the path to the limiting set of labels that should be considered under traversal. // inMorphism, outMorphism, et al should constrain edges by this set. // A nil in this field represents all labels. // // Claimed by the withLabel morphism labelSet shape.Shape } func (c pathContext) copy() pathContext { return pathContext{ labelSet: c.labelSet, } } // Path represents either a morphism (a pre-defined path stored for later use), // or a concrete path, consisting of a morphism and an underlying QuadStore. type Path struct { stack []morphism qs graph.QuadStore // Optionally. A nil qs is equivalent to a morphism. baseContext pathContext } // IsMorphism returns whether this Path is a morphism. func (p *Path) IsMorphism() bool { return p.qs == nil } // StartMorphism creates a new Path with no underlying QuadStore. func StartMorphism(nodes ...quad.Value) *Path { return StartPath(nil, nodes...) } func newPath(qs graph.QuadStore, m ...morphism) *Path { qs = graph.Unwrap(qs) return &Path{ stack: m, qs: qs, } } // StartPath creates a new Path from a set of nodes and an underlying QuadStore. func StartPath(qs graph.QuadStore, nodes ...quad.Value) *Path { return newPath(qs, isMorphism(nodes...)) } // StartPathNodes creates a new Path from a set of nodes and an underlying QuadStore. func StartPathNodes(qs graph.QuadStore, nodes ...graph.Ref) *Path { return newPath(qs, isNodeMorphism(nodes...)) } // PathFromIterator creates a new Path from a set of nodes contained in iterator. func PathFromIterator(qs graph.QuadStore, it iterator.Shape) *Path { return newPath(qs, iteratorMorphism(it)) } // NewPath creates a new, empty Path. func NewPath(qs graph.QuadStore) *Path { return newPath(qs) } // Clone returns a clone of the current path. func (p *Path) Clone() *Path { stack := p.stack return &Path{ stack: stack[:len(stack):len(stack)], qs: p.qs, baseContext: p.baseContext, } } // Unexported clone method returns a *Path with a copy of the original stack, // with assumption that the new stack will be appended to. func (p *Path) clone() *Path { stack := p.stack p.stack = stack[:len(stack):len(stack)] return &Path{ stack: stack, qs: p.qs, baseContext: p.baseContext, } } // Reverse returns a new Path that is the reverse of the current one. func (p *Path) Reverse() *Path { newPath := NewPath(p.qs) ctx := &newPath.baseContext for i := len(p.stack) - 1; i >= 0; i-- { var revMorphism morphism revMorphism, ctx = p.stack[i].Reversal(ctx) newPath.stack = append(newPath.stack, revMorphism) } return newPath } // Is declares that the current nodes in this path are only the nodes // passed as arguments. func (p *Path) Is(nodes ...quad.Value) *Path { np := p.clone() np.stack = append(np.stack, isMorphism(nodes...)) return np } // Regex represents the nodes that are matching provided regexp pattern. // It will only include Raw and String values. func (p *Path) Regex(pattern *regexp.Regexp) *Path { return p.Filters(shape.Regexp{Re: pattern, Refs: false}) } // RegexWithRefs is the same as Regex, but also matches IRIs and BNodes. // // Consider using it carefully. In most cases it's better to reconsider // your graph structure instead of relying on slow unoptimizable regexp. // // An example of incorrect usage is to match IRIs: // // // Via regexp like: // http://example.org/page.* // // The right way is to explicitly link graph nodes and query them by this relation: // func (p *Path) RegexWithRefs(pattern *regexp.Regexp) *Path { return p.Filters(shape.Regexp{Re: pattern, Refs: true}) } // Filter represents the nodes that are passing comparison with provided value. func (p *Path) Filter(op iterator.Operator, node quad.Value) *Path { return p.Filters(shape.Comparison{Op: op, Val: node}) } // Filters represents the nodes that are passing provided filters. func (p *Path) Filters(filters ...shape.ValueFilter) *Path { np := p.clone() np.stack = append(np.stack, filterMorphism(filters)) return np } // Tag adds tag strings to the nodes at this point in the path for each result // path in the set. func (p *Path) Tag(tags ...string) *Path { np := p.clone() np.stack = append(np.stack, tagMorphism(tags...)) return np } // Out updates this Path to represent the nodes that are adjacent to the // current nodes, via the given outbound predicate. // // For example: // // Returns the list of nodes that "B" follows. // // // // Will return []string{"F"} if there is a predicate (edge) from "B" // // to "F" labelled "follows". // StartPath(qs, "A").Out("follows") func (p *Path) Out(via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, outMorphism(nil, via...)) return np } // In updates this Path to represent the nodes that are adjacent to the // current nodes, via the given inbound predicate. // // For example: // // Return the list of nodes that follow "B". // // // // Will return []string{"A", "C", "D"} if there are the appropriate // // edges from those nodes to "B" labelled "follows". // StartPath(qs, "B").In("follows") func (p *Path) In(via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, inMorphism(nil, via...)) return np } // InWithTags is exactly like In, except it tags the value of the predicate // traversed with the tags provided. func (p *Path) InWithTags(tags []string, via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, inMorphism(tags, via...)) return np } // OutWithTags is exactly like In, except it tags the value of the predicate // traversed with the tags provided. func (p *Path) OutWithTags(tags []string, via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, outMorphism(tags, via...)) return np } // Both updates this path following both inbound and outbound predicates. // // For example: // // Return the list of nodes that follow or are followed by "B". // // // // Will return []string{"A", "C", "D", "F} if there are the appropriate // // edges from those nodes to "B" labelled "follows", in either direction. // StartPath(qs, "B").Both("follows") func (p *Path) Both(via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, bothMorphism(nil, via...)) return np } // BothWithTags is exactly like Both, except it tags the value of the predicate // traversed with the tags provided. func (p *Path) BothWithTags(tags []string, via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, bothMorphism(tags, via...)) return np } // Labels updates this path to represent the nodes of the labels // of inbound and outbound quads. func (p *Path) Labels() *Path { np := p.clone() np.stack = append(np.stack, labelsMorphism()) return np } // InPredicates updates this path to represent the nodes of the valid inbound // predicates from the current nodes. // // For example: // // Returns a list of predicates valid from "bob" // // // // Will return []string{"follows"} if there are any things that "follow" Bob // StartPath(qs, "bob").InPredicates() func (p *Path) InPredicates() *Path { np := p.clone() np.stack = append(np.stack, predicatesMorphism(true)) return np } // OutPredicates updates this path to represent the nodes of the valid inbound // predicates from the current nodes. // // For example: // // Returns a list of predicates valid from "bob" // // // // Will return []string{"follows", "status"} if there are edges from "bob" // // labelled "follows", and edges from "bob" that describe his "status". // StartPath(qs, "bob").OutPredicates() func (p *Path) OutPredicates() *Path { np := p.clone() np.stack = append(np.stack, predicatesMorphism(false)) return np } // SavePredicates saves either forward or reverse predicates of current node // without changing path location. func (p *Path) SavePredicates(rev bool, tag string) *Path { np := p.clone() np.stack = append(np.stack, savePredicatesMorphism(rev, tag)) return np } // And updates the current Path to represent the nodes that match both the // current Path so far, and the given Path. func (p *Path) And(path *Path) *Path { np := p.clone() np.stack = append(np.stack, andMorphism(path)) return np } // Optional adds an optional path to evaluate. This path will only contribute to tags and won't change iteration results. func (p *Path) Optional(path *Path) *Path { np := p.clone() np.stack = append(np.stack, andOptMorphism(path.Reverse())) return np } // Or updates the current Path to represent the nodes that match either the // current Path so far, or the given Path. func (p *Path) Or(path *Path) *Path { np := p.clone() np.stack = append(np.stack, orMorphism(path)) return np } // Except updates the current Path to represent the all of the current nodes // except those in the supplied Path. // // For example: // // Will return []string{"B"} // StartPath(qs, "A", "B").Except(StartPath(qs, "A")) func (p *Path) Except(path *Path) *Path { np := p.clone() np.stack = append(np.stack, exceptMorphism(path)) return np } // Unique updates the current Path to contain only unique nodes. func (p *Path) Unique() *Path { np := p.clone() np.stack = append(np.stack, uniqueMorphism()) return np } // Follow allows you to stitch two paths together. The resulting path will start // from where the first path left off and continue iterating down the path given. func (p *Path) Follow(path *Path) *Path { np := p.clone() np.stack = append(np.stack, followMorphism(path)) return np } // FollowReverse is the same as follow, except it will iterate backwards up the // path given as argument. func (p *Path) FollowReverse(path *Path) *Path { np := p.clone() np.stack = append(np.stack, followMorphism(path.Reverse())) return np } // FollowRecursive will repeatedly follow the given string predicate or Path // object starting from the given node(s), through the morphism or pattern // provided, ignoring loops. For example, this turns "parent" into "all // ancestors", by repeatedly following the "parent" connection on the result of // the parent nodes. // // The second argument, "maxDepth" is the maximum number of recursive steps before // stopping and returning. // If -1 is passed, it will have no limit. // If 0 is passed, it will use the default value of 50 steps before returning. // If 1 is passed, it will stop after 1 step before returning, and so on. // // The third argument, "depthTags" is a set of tags that will return strings of // numeric values relating to how many applications of the path were applied the // first time the result node was seen. // // This is a very expensive operation in practice. Be sure to use it wisely. func (p *Path) FollowRecursive(via interface{}, maxDepth int, depthTags []string) *Path { var path *Path switch v := via.(type) { case string: path = StartMorphism().Out(v) case quad.Value: path = StartMorphism().Out(v) case *Path: path = v default: panic("did not pass a string predicate or a Path to FollowRecursive") } np := p.clone() np.stack = append(p.stack, followRecursiveMorphism(path, maxDepth, depthTags)) return np } // Save will, from the current nodes in the path, retrieve the node // one linkage away (given by either a path or a predicate), add the given // tag, and propagate that to the result set. // // For example: // // Will return []map[string]string{{"social_status: "cool"}} // StartPath(qs, "B").Save("status", "social_status" func (p *Path) Save(via interface{}, tag string) *Path { np := p.clone() np.stack = append(np.stack, saveMorphism(via, tag)) return np } // SaveReverse is the same as Save, only in the reverse direction // (the subject of the linkage should be tagged, instead of the object). func (p *Path) SaveReverse(via interface{}, tag string) *Path { np := p.clone() np.stack = append(np.stack, saveReverseMorphism(via, tag)) return np } // SaveOptional is the same as Save, but does not require linkage to exist. func (p *Path) SaveOptional(via interface{}, tag string) *Path { np := p.clone() np.stack = append(np.stack, saveOptionalMorphism(via, tag)) return np } // SaveOptionalReverse is the same as SaveReverse, but does not require linkage to exist. func (p *Path) SaveOptionalReverse(via interface{}, tag string) *Path { np := p.clone() np.stack = append(np.stack, saveOptionalReverseMorphism(via, tag)) return np } // HasPath limits the paths to be ones where the current nodes have a given subpath. func (p *Path) HasPath(p2 *Path) *Path { np := p.clone() np.stack = append(np.stack, hasPathMorphism(p2.Reverse())) return np } // Has limits the paths to be ones where the current nodes have some linkage // to some known node. func (p *Path) Has(via interface{}, nodes ...quad.Value) *Path { np := p.clone() np.stack = append(np.stack, hasMorphism(via, false, nodes...)) return np } // HasReverse limits the paths to be ones where some known node have some linkage // to the current nodes. func (p *Path) HasReverse(via interface{}, nodes ...quad.Value) *Path { np := p.clone() np.stack = append(np.stack, hasMorphism(via, true, nodes...)) return np } // HasFilter limits the paths to be ones where the current nodes have some linkage // to some nodes that pass provided filters. func (p *Path) HasFilter(via interface{}, rev bool, filt ...shape.ValueFilter) *Path { np := p.clone() np.stack = append(np.stack, hasFilterMorphism(via, rev, filt)) return np } // LabelContext restricts the following operations (such as In, Out) to only // traverse edges that match the given set of labels. func (p *Path) LabelContext(via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, labelContextMorphism(nil, via...)) return np } // LabelContextWithTags is exactly like LabelContext, except it tags the value // of the label used in the traversal with the tags provided. func (p *Path) LabelContextWithTags(tags []string, via ...interface{}) *Path { np := p.clone() np.stack = append(np.stack, labelContextMorphism(tags, via...)) return np } // Back returns to a previously tagged place in the path. Any constraints applied after the Tag will remain in effect, but traversal continues from the tagged point instead, not from the end of the chain. // // For example: // // Will return "bob" iff "bob" is cool // StartPath(qs, "bob").Tag("person_tag").Out("status").Is("cool").Back("person_tag") func (p *Path) Back(tag string) *Path { newPath := NewPath(p.qs) i := len(p.stack) - 1 ctx := &newPath.baseContext for { if i < 0 { return p.Reverse() } if p.stack[i].IsTag { for _, x := range p.stack[i].tags { if x == tag { // Found what we're looking for. p.stack = p.stack[:i+1] return p.And(newPath) } } } var revMorphism morphism revMorphism, ctx = p.stack[i].Reversal(ctx) newPath.stack = append(newPath.stack, revMorphism) i-- } } // BuildIterator returns an iterator from this given Path. Note that you must // call this with a full path (not a morphism), since a morphism does not have // the ability to fetch the underlying quads. This function will panic if // called with a morphism (i.e. if p.IsMorphism() is true). func (p *Path) BuildIterator(ctx context.Context) iterator.Shape { if p.IsMorphism() { panic("Building an iterator from a morphism. Bind a QuadStore with BuildIteratorOn(qs)") } return p.BuildIteratorOn(ctx, p.qs) } // BuildIteratorOn will return an iterator for this path on the given QuadStore. func (p *Path) BuildIteratorOn(ctx context.Context, qs graph.QuadStore) iterator.Shape { return shape.BuildIterator(ctx, qs, p.Shape()) } // MorphismFor returns the morphism of this path. The returned value is a // function that, when given an existing Iterator, will return a new Iterator // that yields the subset of values from the existing iterator matched by the // current Path. func (p *Path) MorphismFor(qs graph.QuadStore) iterator.Morphism { return func(it iterator.Shape) iterator.Shape { return p.ShapeFrom(&iteratorShape{it: it}).BuildIterator(qs) } } // Skip will omit a number of values from result set. func (p *Path) Skip(v int64) *Path { p.stack = append(p.stack, skipMorphism(v)) return p } func (p *Path) Order() *Path { p.stack = append(p.stack, orderMorphism()) return p } // Limit will limit a number of values in result set. func (p *Path) Limit(v int64) *Path { p.stack = append(p.stack, limitMorphism(v)) return p } // Count will count a number of results as it's own result set. func (p *Path) Count() *Path { p.stack = append(p.stack, countMorphism()) return p } // Iterate is an shortcut for graph.Iterate. func (p *Path) Iterate(ctx context.Context) *iterator.Chain { return shape.Iterate(ctx, p.qs, p.Shape()) } func (p *Path) Shape() shape.Shape { return p.ShapeFrom(shape.AllNodes{}) } func (p *Path) ShapeFrom(from shape.Shape) shape.Shape { s := from ctx := &p.baseContext for _, m := range p.stack { s, ctx = m.Apply(s, ctx) } return s } ================================================ FILE: query/path/path_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package path_test import ( "testing" _ "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/query/path/pathtest" ) func TestMorphisms(t *testing.T) { pathtest.RunTestMorphisms(t, nil) } ================================================ FILE: query/path/pathtest/pathtest.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pathtest import ( "context" "reflect" "regexp" "sort" "testing" "time" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphtest/testutil" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/cayley/query/shape" _ "github.com/cayleygraph/cayley/writer" ) // This is a simple test graph. // // +-------+ +------+ // | alice |----- ->| fred |<-- // +-------+ \---->+-------+-/ +------+ \-+-------+ // ----->| #bob# | | | emily | // +---------+--/ --->+-------+ | +-------+ // | charlie | / v // +---------+ / +--------+ // \--- +--------+ | #greg# | // \-->| #dani# |------------>+--------+ // +--------+ func makeTestStore(t testing.TB, fnc testutil.DatabaseFunc, quads ...quad.Quad) graph.QuadStore { if len(quads) == 0 { quads = testutil.LoadGraph(t, "data/testdata.nq") } var ( qs graph.QuadStore opts graph.Options ) if fnc != nil { qs, opts = fnc(t) } else { qs, _ = graph.NewQuadStore("memstore", "", nil) } _ = testutil.MakeWriter(t, qs, opts, quads...) return qs } func runTopLevel(qs graph.QuadStore, path *path.Path, opt bool) ([]quad.Value, error) { pb := path.Iterate(context.TODO()) if !opt { pb = pb.UnOptimized() } return pb.Paths(false).AllValues(qs) } func runTag(qs graph.QuadStore, path *path.Path, tag string, opt, keepEmpty bool) ([]quad.Value, error) { var out []quad.Value pb := path.Iterate(context.TODO()) if !opt { pb = pb.UnOptimized() } err := pb.Paths(true).TagEach(func(tags map[string]graph.Ref) error { if t, ok := tags[tag]; ok { tv, err := qs.NameOf(t) if err != nil { return err } out = append(out, tv) } else if keepEmpty { out = append(out, vEmpty) } return nil }) return out, err } func runAllTags(qs graph.QuadStore, path *path.Path, opt bool) ([]map[string]quad.Value, error) { var out []map[string]quad.Value pb := path.Iterate(context.TODO()) if !opt { pb = pb.UnOptimized() } err := pb.Paths(true).TagValues(qs, func(tags map[string]quad.Value) error { out = append(out, tags) return nil }) return out, err } type test struct { skip bool message string path *path.Path expect []quad.Value expectAlt [][]quad.Value tag string unsorted bool empty bool // do not skip empty tags } // Define morphisms without a QuadStore const ( vEmpty = quad.String("") vFollows = quad.IRI("follows") vAre = quad.IRI("are") vStatus = quad.IRI("status") vPredicate = quad.IRI("predicates") vCool = quad.String("cool_person") vSmart = quad.String("smart_person") vSmartGraph = quad.IRI("smart_graph") vAlice = quad.IRI("alice") vBob = quad.IRI("bob") vCharlie = quad.IRI("charlie") vDani = quad.IRI("dani") vFred = quad.IRI("fred") vGreg = quad.IRI("greg") vEmily = quad.IRI("emily") ) var ( grandfollows = path.StartMorphism().Out(vFollows).Out(vFollows) ) func testSet(qs graph.QuadStore) []test { return []test{ { message: "out", path: path.StartPath(qs, vAlice).Out(vFollows), expect: []quad.Value{vBob}, }, { message: "out (any)", path: path.StartPath(qs, vBob).Out(), expect: []quad.Value{vFred, vCool}, }, { message: "out (raw)", path: path.StartPath(qs, quad.Raw(vAlice.String())).Out(quad.Raw(vFollows.String())), expect: []quad.Value{vBob}, }, { message: "in", path: path.StartPath(qs, vBob).In(vFollows), expect: []quad.Value{vAlice, vCharlie, vDani}, }, { message: "in (any)", path: path.StartPath(qs, vBob).In(), expect: []quad.Value{vAlice, vCharlie, vDani}, }, { message: "filter nodes", path: path.StartPath(qs).Filter(iterator.CompareGT, quad.IRI("p")), expect: []quad.Value{vPredicate, vSmartGraph, vStatus}, }, { message: "in with filter", path: path.StartPath(qs, vBob).In(vFollows).Filter(iterator.CompareGT, quad.IRI("c")), expect: []quad.Value{vCharlie, vDani}, }, { message: "in with regex", path: path.StartPath(qs, vBob).In(vFollows).Regex(regexp.MustCompile("ar?li.*e")), expect: nil, }, { message: "in with regex (include IRIs)", path: path.StartPath(qs, vBob).In(vFollows).RegexWithRefs(regexp.MustCompile("ar?li.*e")), expect: []quad.Value{vAlice, vCharlie}, }, { message: "path Out", path: path.StartPath(qs, vBob).Out(path.StartPath(qs, vPredicate).Out(vAre)), expect: []quad.Value{vFred, vCool}, }, { message: "path Out (raw)", path: path.StartPath(qs, quad.Raw(vBob.String())).Out(path.StartPath(qs, quad.Raw(vPredicate.String())).Out(quad.Raw(vAre.String()))), expect: []quad.Value{vFred, vCool}, }, { message: "And", path: path.StartPath(qs, vDani).Out(vFollows).And( path.StartPath(qs, vCharlie).Out(vFollows)), expect: []quad.Value{vBob}, }, { message: "Or", path: path.StartPath(qs, vFred).Out(vFollows).Or( path.StartPath(qs, vAlice).Out(vFollows)), expect: []quad.Value{vBob, vGreg}, }, { message: "implicit All", path: path.StartPath(qs), expect: []quad.Value{vAlice, vBob, vCharlie, vDani, vEmily, vFred, vGreg, vFollows, vStatus, vCool, vPredicate, vAre, vSmartGraph, vSmart}, }, { message: "follow", path: path.StartPath(qs, vCharlie).Follow(path.StartMorphism().Out(vFollows).Out(vFollows)), expect: []quad.Value{vBob, vFred, vGreg}, }, { message: "followR", path: path.StartPath(qs, vFred).FollowReverse(path.StartMorphism().Out(vFollows).Out(vFollows)), expect: []quad.Value{vAlice, vCharlie, vDani}, }, { message: "is, tag, instead of FollowR", path: path.StartPath(qs).Tag("first").Follow(path.StartMorphism().Out(vFollows).Out(vFollows)).Is(vFred), expect: []quad.Value{vAlice, vCharlie, vDani}, tag: "first", }, { message: "Except to filter out a single vertex", path: path.StartPath(qs, vAlice, vBob).Except(path.StartPath(qs, vAlice)), expect: []quad.Value{vBob}, }, { message: "chained Except", path: path.StartPath(qs, vAlice, vBob, vCharlie).Except(path.StartPath(qs, vBob)).Except(path.StartPath(qs, vAlice)), expect: []quad.Value{vCharlie}, }, { message: "Unique", path: path.StartPath(qs, vAlice, vBob, vCharlie).Out(vFollows).Unique(), expect: []quad.Value{vBob, vDani, vFred}, }, { message: "simple save", path: path.StartPath(qs).Save(vStatus, "somecool"), tag: "somecool", expect: []quad.Value{vCool, vCool, vCool, vSmart, vSmart}, }, { message: "simple saveR", path: path.StartPath(qs, vCool).SaveReverse(vStatus, "who"), tag: "who", expect: []quad.Value{vGreg, vDani, vBob}, }, { message: "save with a next path", path: path.StartPath(qs, vDani, vBob).Save(vFollows, "target"), tag: "target", expect: []quad.Value{vBob, vFred, vGreg}, }, { message: "save all with a next path", path: path.StartPath(qs).Save(vFollows, "target"), tag: "target", expect: []quad.Value{vBob, vBob, vBob, vDani, vFred, vFred, vGreg, vGreg}, }, { message: "simple Has", path: path.StartPath(qs).Has(vStatus, vCool), expect: []quad.Value{vGreg, vDani, vBob}, }, { message: "filter nodes with has", path: path.StartPath(qs).HasFilter(vFollows, false, shape.Comparison{ Op: iterator.CompareGT, Val: quad.IRI("f"), }), expect: []quad.Value{vBob, vDani, vEmily, vFred}, }, { message: "has path", path: path.StartPath(qs).HasPath(path.StartMorphism().Out(vStatus).Is(vCool)), expect: []quad.Value{vGreg, vDani, vBob}, }, { message: "string prefix", path: path.StartPath(qs).Filters(shape.Wildcard{ Pattern: `bo%`, }), expect: []quad.Value{vBob}, }, { message: "three letters and range", path: path.StartPath(qs).Filters(shape.Wildcard{ Pattern: `???`, }, shape.Comparison{ Op: iterator.CompareGT, Val: quad.IRI("b"), }), expect: []quad.Value{vBob}, }, { message: "part in string", path: path.StartPath(qs).Filters(shape.Wildcard{ Pattern: `%ed%`, }), expect: []quad.Value{vFred, vPredicate}, }, { message: "Limit", path: path.StartPath(qs).Has(vStatus, vCool).Limit(2), // TODO(dennwc): resolve this ordering issue expectAlt: [][]quad.Value{ {vBob, vGreg}, {vBob, vDani}, {vDani, vGreg}, }, }, { message: "Skip", path: path.StartPath(qs).Has(vStatus, vCool).Skip(2), expectAlt: [][]quad.Value{ {vBob}, {vDani}, {vGreg}, }, }, { message: "Skip and Limit", path: path.StartPath(qs).Has(vStatus, vCool).Skip(1).Limit(1), expectAlt: [][]quad.Value{ {vBob}, {vDani}, {vGreg}, }, }, { message: "Count", path: path.StartPath(qs).Has(vStatus).Count(), expect: []quad.Value{quad.Int(5)}, }, { message: "double Has", path: path.StartPath(qs).Has(vStatus, vCool).Has(vFollows, vFred), expect: []quad.Value{vBob}, }, { message: "simple HasReverse", path: path.StartPath(qs).HasReverse(vStatus, vBob), expect: []quad.Value{vCool}, }, { message: ".Tag()-.Is()-.Back()", path: path.StartPath(qs, vBob).In(vFollows).Tag("foo").Out(vStatus).Is(vCool).Back("foo"), expect: []quad.Value{vDani}, }, { message: "do multiple .Back()s", path: path.StartPath(qs, vEmily).Out(vFollows).Tag("f").Out(vFollows).Out(vStatus).Is(vCool).Back("f").In(vFollows).In(vFollows).Tag("acd").Out(vStatus).Is(vCool).Back("f"), tag: "acd", expect: []quad.Value{vDani}, }, { message: "Labels()", path: path.StartPath(qs, vGreg).Labels(), expect: []quad.Value{vSmartGraph}, }, { message: "InPredicates()", path: path.StartPath(qs, vBob).InPredicates(), expect: []quad.Value{vFollows}, }, { message: "OutPredicates()", path: path.StartPath(qs, vBob).OutPredicates(), expect: []quad.Value{vFollows, vStatus}, }, { message: "SavePredicates(in)", path: path.StartPath(qs, vBob).SavePredicates(true, "pred"), expect: []quad.Value{vFollows, vFollows, vFollows}, tag: "pred", }, { message: "SavePredicates(out)", path: path.StartPath(qs, vBob).SavePredicates(false, "pred"), expect: []quad.Value{vFollows, vStatus}, tag: "pred", }, // Morphism tests { message: "simple morphism", path: path.StartPath(qs, vCharlie).Follow(grandfollows), expect: []quad.Value{vGreg, vFred, vBob}, }, { message: "reverse morphism", path: path.StartPath(qs, vFred).FollowReverse(grandfollows), expect: []quad.Value{vAlice, vCharlie, vDani}, }, // Context tests { message: "query without label limitation", path: path.StartPath(qs, vGreg).Out(vStatus), expect: []quad.Value{vSmart, vCool}, }, { message: "query with label limitation", path: path.StartPath(qs, vGreg).LabelContext(vSmartGraph).Out(vStatus), expect: []quad.Value{vSmart}, }, { message: "reverse context", path: path.StartPath(qs, vGreg).Tag("base").LabelContext(vSmartGraph).Out(vStatus).Tag("status").Back("base"), expect: []quad.Value{vGreg}, }, // Optional tests { message: "save limits top level", path: path.StartPath(qs, vBob, vCharlie).Out(vFollows).Save(vStatus, "statustag"), expect: []quad.Value{vBob, vDani}, }, { message: "optional still returns top level", path: path.StartPath(qs, vBob, vCharlie).Out(vFollows).SaveOptional(vStatus, "statustag"), expect: []quad.Value{vBob, vFred, vDani}, }, { message: "optional has the appropriate tags", path: path.StartPath(qs, vBob, vCharlie).Out(vFollows).SaveOptional(vStatus, "statustag"), tag: "statustag", expect: []quad.Value{vCool, vCool}, }, { message: "composite paths (clone paths)", path: func() *path.Path { alicePath := path.StartPath(qs, vAlice) _ = alicePath.Out(vFollows) return alicePath }(), expect: []quad.Value{vAlice}, }, { message: "follow recursive", path: path.StartPath(qs, vCharlie).FollowRecursive(vFollows, 0, nil), expect: []quad.Value{vBob, vDani, vFred, vGreg}, }, { message: "follow recursive (limit depth)", path: path.StartPath(qs, vCharlie).FollowRecursive(vFollows, 1, nil), expect: []quad.Value{vBob, vDani}, }, { message: "find non-existent", path: path.StartPath(qs, quad.IRI("")), expect: nil, }, { message: "use order", path: path.StartPath(qs).Order(), expect: []quad.Value{ vAlice, vAre, vBob, vCharlie, vDani, vEmily, vFollows, vFred, vGreg, vPredicate, vSmartGraph, vStatus, vCool, vSmart, }, }, { message: "use order tags", path: path.StartPath(qs).Tag("target").Order(), tag: "target", expect: []quad.Value{ vAlice, vAre, vBob, vCharlie, vDani, vEmily, vFollows, vFred, vGreg, vPredicate, vSmartGraph, vStatus, vCool, vSmart, }, }, { message: "order with a next path", path: path.StartPath(qs, vDani, vBob).Save(vFollows, "target").Order(), tag: "target", expect: []quad.Value{vBob, vFred, vGreg}, }, { message: "order with a next path", path: path.StartPath(qs).Order().Has(vFollows, vBob), expect: []quad.Value{vAlice, vCharlie, vDani}, unsorted: true, skip: true, // TODO(dennwc): optimize Order in And properly }, { message: "optional path", path: path.StartPath(qs, vBob, vDani, vFred).Optional(path.StartMorphism().Save(vStatus, "status")), tag: "status", empty: true, expect: []quad.Value{vEmpty, vCool, vCool}, }, } } func RunTestMorphisms(t *testing.T, fnc testutil.DatabaseFunc) { for _, ftest := range []func(*testing.T, testutil.DatabaseFunc){ testFollowRecursive, testFollowRecursiveHas, } { ftest(t, fnc) } qs := makeTestStore(t, fnc) for _, test := range testSet(qs) { for _, opt := range []bool{true, false} { name := test.message if !opt { name += " (unoptimized)" } t.Run(name, func(t *testing.T) { if test.skip { t.SkipNow() } var ( got []quad.Value err error ) start := time.Now() if test.tag == "" { got, err = runTopLevel(qs, test.path, opt) } else { got, err = runTag(qs, test.path, test.tag, opt, test.empty) } dt := time.Since(start) if err != nil { t.Error(err) return } if !test.unsorted { sort.Sort(quad.ByValueString(got)) } var eq bool exp := test.expect if test.expectAlt != nil { for _, alt := range test.expectAlt { exp = alt if !test.unsorted { sort.Sort(quad.ByValueString(exp)) } eq = reflect.DeepEqual(got, exp) if eq { break } } } else { if !test.unsorted { sort.Sort(quad.ByValueString(test.expect)) } eq = reflect.DeepEqual(got, test.expect) } if !eq { t.Errorf("got: %v(%d) expected: %v(%d)", got, len(got), exp, len(exp)) } else { t.Logf("%12v %v", dt, name) } }) } } } func testFollowRecursive(t *testing.T, fnc testutil.DatabaseFunc) { qs := makeTestStore(t, fnc, []quad.Quad{ quad.MakeIRI("a", "parent", "b", ""), quad.MakeIRI("b", "parent", "c", ""), quad.MakeIRI("c", "parent", "d", ""), quad.MakeIRI("c", "labels", "tag", ""), quad.MakeIRI("d", "parent", "e", ""), quad.MakeIRI("d", "labels", "tag", ""), }...) qu := path.StartPath(qs, quad.IRI("a")).FollowRecursive( path.StartMorphism().Out(quad.IRI("parent")), 0, nil, ).Has(quad.IRI("labels"), quad.IRI("tag")) expect := []quad.Value{quad.IRI("c"), quad.IRI("d")} const msg = "follows recursive order" for _, opt := range []bool{true, false} { unopt := "" if !opt { unopt = " (unoptimized)" } t.Run(msg+unopt, func(t *testing.T) { got, err := runTopLevel(qs, qu, opt) if err != nil { t.Errorf("Failed to check %s%s: %v", msg, unopt, err) return } sort.Sort(quad.ByValueString(got)) sort.Sort(quad.ByValueString(expect)) if !reflect.DeepEqual(got, expect) { t.Errorf("Failed to %s%s, got: %v(%d) expected: %v(%d)", msg, unopt, got, len(got), expect, len(expect)) } }) } } type byTags struct { tags []string arr []map[string]quad.Value } func (b byTags) Len() int { return len(b.arr) } func (b byTags) Less(i, j int) bool { m1, m2 := b.arr[i], b.arr[j] for _, t := range b.tags { v1, v2 := m1[t], m2[t] s1, s2 := quad.ToString(v1), quad.ToString(v2) if s1 < s2 { return true } else if s1 > s2 { return false } } return false } func (b byTags) Swap(i, j int) { b.arr[i], b.arr[j] = b.arr[j], b.arr[i] } func testFollowRecursiveHas(t *testing.T, fnc testutil.DatabaseFunc) { qs := makeTestStore(t, fnc, []quad.Quad{ quad.MakeIRI("1", "relatesTo", "x", ""), quad.MakeIRI("2", "relatesTo", "x", ""), quad.MakeIRI("3", "relatesTo", "y", ""), quad.MakeIRI("1", "knows", "2", ""), quad.MakeIRI("2", "knows", "3", ""), quad.MakeIRI("2", "knows", "1", ""), }...) qu := path.StartPath(qs, quad.IRI("1")).FollowRecursive( path.StartMorphism().Tag("pid").Out(quad.IRI("knows")), 2, nil, ).Has(quad.IRI("relatesTo")).Tag("id") expect := []map[string]quad.Value{ {"id": quad.IRI("1"), "pid": quad.IRI("2")}, {"id": quad.IRI("2"), "pid": quad.IRI("1")}, {"id": quad.IRI("3"), "pid": quad.IRI("2")}, } sortTags := []string{"id", "pid"} sort.Sort(byTags{ tags: sortTags, arr: expect, }) const msg = "follows recursive loop" for _, opt := range []bool{true, false} { unopt := "" if !opt { unopt = " (unoptimized)" } t.Run(msg+unopt, func(t *testing.T) { got, err := runAllTags(qs, qu, opt) if err != nil { t.Errorf("Failed to check %s%s: %v", msg, unopt, err) return } sort.Sort(byTags{ tags: sortTags, arr: got, }) require.Equal(t, expect, got) }) } } ================================================ FILE: query/session.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package query defines the graph session interface general to all query languages. package query import ( "context" "errors" "fmt" "io" "github.com/cayleygraph/cayley/graph" ) var ErrParseMore = errors.New("query: more input required") type ErrUnsupportedCollation struct { Collation Collation } func (e *ErrUnsupportedCollation) Error() string { return fmt.Sprintf("unsupported collation: %v", e.Collation) } // Iterator for query results. type Iterator interface { // Next advances the iterator to the next value, which will then be available through // the Result method. It returns false if no further advancement is possible, or if an // error was encountered during iteration. Err should be consulted to distinguish // between the two cases. Next(ctx context.Context) bool // Results returns the current result. The type depends on the collation mode of the query. Result() interface{} // Err returns any error that was encountered by the Iterator. Err() error // Close the iterator and do internal cleanup. Close() error } // Collation of results. type Collation int const ( // Raw collates results as maps or arrays of graph.Refs or any other query-native or graph-native data type. Raw = Collation(iota) // REPL collates results as strings which will be used in CLI. REPL = Collation(iota) // JSON collates results as maps, arrays and values, that can be encoded to JSON. JSON // JSONLD collates results as maps, arrays and values compatible with JSON-LD spec. JSONLD ) // Options for the query execution. type Options struct { Limit int Collation Collation } type Session interface { // Execute runs the query and returns an iterator over the results. // Type of results depends on Collation. See Options for details. Execute(ctx context.Context, query string, opt Options) (Iterator, error) } type REPLSession = Session // ResponseWriter is a subset of http.ResponseWriter type ResponseWriter interface { Write([]byte) (int, error) WriteHeader(int) } // Language is a description of query language. type Language struct { Name string Session func(graph.QuadStore) Session // Custom HTTP handlers HTTPQuery func(ctx context.Context, qs graph.QuadStore, w ResponseWriter, r io.Reader) HTTPError func(w ResponseWriter, err error) } var languages = make(map[string]Language) // RegisterLanguage register a new query language. func RegisterLanguage(lang Language) { languages[lang.Name] = lang } // NewSession creates a new session for specified query language. // It returns nil if language was not registered. func NewSession(qs graph.QuadStore, lang string) Session { if l := languages[lang]; l.Session != nil { return l.Session(qs) } return nil } // GetLanguage returns a query language description. // It returns nil if language was not registered. func GetLanguage(lang string) *Language { l, ok := languages[lang] if ok { return &l } return nil } // Languages returns names of registered query languages. func Languages() []string { out := make([]string, 0, len(languages)) for name := range languages { out = append(out, name) } return out } // Execute runs the query in a given language and returns an iterator over the results. // Type of results depends on Collation. See Options for details. func Execute(ctx context.Context, qs graph.QuadStore, lang, query string, opt Options) (Iterator, error) { l := GetLanguage(lang) if l == nil { return nil, fmt.Errorf("unsupported language: %q", lang) } sess := l.Session(qs) return sess.Execute(ctx, query, opt) } ================================================ FILE: query/sexp/parser.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package sexp import ( "context" "github.com/badgerodon/peg" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) func BuildIteratorTreeForQuery(ctx context.Context, qs graph.QuadStore, query string) iterator.Shape { s, err := BuildShape(ctx, query) if err != nil { return iterator.NewError(err) } return shape.BuildIterator(ctx, qs, s) } func BuildShape(ctx context.Context, query string) (shape.Shape, error) { tree := parseQuery(query) s, _ := buildShape(tree) s, _ = shape.Optimize(ctx, s, nil) return s, nil } func ParseString(input string) string { return parseQuery(input).String() } func newParser() *peg.Parser { parser := peg.NewParser() start := parser.NonTerminal("Start") whitespace := parser.NonTerminal("Whitespace") quotedString := parser.NonTerminal("QuotedString") rootConstraint := parser.NonTerminal("RootConstraint") constraint := parser.NonTerminal("Constraint") colonIdentifier := parser.NonTerminal("ColonIdentifier") variable := parser.NonTerminal("Variable") identifier := parser.NonTerminal("Identifier") fixedNode := parser.NonTerminal("FixedNode") nodeIdent := parser.NonTerminal("NodeIdentifier") predIdent := parser.NonTerminal("PredIdentifier") reverse := parser.NonTerminal("Reverse") predKeyword := parser.NonTerminal("PredicateKeyword") optional := parser.NonTerminal("OptionalKeyword") start.Expression = rootConstraint whitespace.Expression = parser.OneOrMore( parser.OrderedChoice( parser.Terminal(' '), parser.Terminal('\t'), parser.Terminal('\n'), parser.Terminal('\r'), ), ) quotedString.Expression = parser.Sequence( parser.Terminal('"'), parser.OneOrMore( parser.OrderedChoice( parser.Range('0', '9'), parser.Range('a', 'z'), parser.Range('A', 'Z'), parser.Terminal('_'), parser.Terminal('/'), parser.Terminal(':'), parser.Terminal(' '), parser.Terminal('\''), ), ), parser.Terminal('"'), ) predKeyword.Expression = parser.OrderedChoice( optional, ) optional.Expression = parser.Sequence( parser.Terminal('o'), parser.Terminal('p'), parser.Terminal('t'), parser.Terminal('i'), parser.Terminal('o'), parser.Terminal('n'), parser.Terminal('a'), parser.Terminal('l'), ) identifier.Expression = parser.OneOrMore( parser.OrderedChoice( parser.Range('0', '9'), parser.Range('a', 'z'), parser.Range('A', 'Z'), parser.Terminal('_'), parser.Terminal('.'), parser.Terminal('/'), parser.Terminal(':'), parser.Terminal('#'), ), ) reverse.Expression = parser.Terminal('!') variable.Expression = parser.Sequence( parser.Terminal('$'), identifier, ) colonIdentifier.Expression = parser.Sequence( parser.Terminal(':'), identifier, ) fixedNode.Expression = parser.OrderedChoice( colonIdentifier, quotedString, ) nodeIdent.Expression = parser.OrderedChoice( variable, fixedNode, ) predIdent.Expression = parser.Sequence( parser.Optional(reverse), parser.OrderedChoice( nodeIdent, constraint, ), ) constraint.Expression = parser.Sequence( parser.Terminal('('), parser.Optional(whitespace), predIdent, parser.Optional(whitespace), parser.Optional(predKeyword), parser.Optional(whitespace), parser.OrderedChoice( nodeIdent, rootConstraint, ), parser.Optional(whitespace), parser.Terminal(')'), ) rootConstraint.Expression = parser.Sequence( parser.Terminal('('), parser.Optional(whitespace), nodeIdent, parser.Optional(whitespace), parser.ZeroOrMore(parser.Sequence( constraint, parser.Optional(whitespace), )), parser.Terminal(')'), ) return parser } func parseQuery(input string) *peg.ExpressionTree { return newParser().Parse(input) } func getIdentString(tree *peg.ExpressionTree) string { out := "" if len(tree.Children) > 0 { for _, child := range tree.Children { out += getIdentString(child) } } else { if tree.Value != '"' { out += string([]rune{rune(tree.Value)}) } } return out } func lookup(s string) shape.Shape { return shape.Lookup{quad.StringToValue(s)} } func buildShape(tree *peg.ExpressionTree) (_ shape.Shape, opt bool) { switch tree.Name { case "Start": return buildShape(tree.Children[0]) case "NodeIdentifier": var out shape.Shape nodeID := getIdentString(tree) if tree.Children[0].Name == "Variable" { out = shape.Save{ From: shape.AllNodes{}, Tags: []string{nodeID}, } } else { n := nodeID if tree.Children[0].Children[0].Name == "ColonIdentifier" { n = nodeID[1:] } out = lookup(n) } return out, false case "PredIdentifier": i := 0 if tree.Children[0].Name == "Reverse" { //Taken care of below i++ } it, _ := buildShape(tree.Children[i]) return shape.Quads{ {Dir: quad.Predicate, Values: it}, }, false case "RootConstraint": var and shape.IntersectOpt for _, c := range tree.Children { switch c.Name { case "NodeIdentifier": fallthrough case "Constraint": it, opt := buildShape(c) if opt { and.AddOptional(it) } else { and.Add(it) } continue default: continue } } return and, false case "Constraint": topLevelDir := quad.Subject subItDir := quad.Object var subAnd shape.IntersectOpt isOptional := false for _, c := range tree.Children { switch c.Name { case "PredIdentifier": if c.Children[0].Name == "Reverse" { topLevelDir = quad.Object subItDir = quad.Subject } it, opt := buildShape(c) if opt { subAnd.AddOptional(it) } else { subAnd.Add(it) } continue case "PredicateKeyword": switch c.Children[0].Name { case "OptionalKeyword": isOptional = true } case "NodeIdentifier": fallthrough case "RootConstraint": it, opt := buildShape(c) l := shape.Quads{ {Dir: subItDir, Values: it}, } if opt { subAnd.AddOptional(l) } else { subAnd.Add(l) } continue default: continue } } return shape.NodesFrom{ Dir: topLevelDir, Quads: subAnd, }, isOptional default: return shape.Null{}, false } } ================================================ FILE: query/sexp/parser_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package sexp import ( "context" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/graph/graphtest/testutil" _ "github.com/cayleygraph/cayley/graph/memstore" sh "github.com/cayleygraph/cayley/query/shape" _ "github.com/cayleygraph/cayley/writer" "github.com/stretchr/testify/require" ) func TestBadParse(t *testing.T) { str := ParseString("()") if str != "" { t.Errorf("Unexpected parse result, got:%q", str) } } var ( quads1 = []quad.Quad{quad.Make("i", "can", "win", nil)} ) var testQueries = []struct { message string add []quad.Quad query string shape sh.Shape expect string tags map[string]string }{ { message: "empty", query: "()", shape: sh.Null{}, }, { message: "get a single quad linkage", add: quads1, query: "($a (:can \"win\"))", shape: sh.Save{ Tags: []string{"$a"}, From: sh.NodesFrom{ Dir: quad.Subject, Quads: sh.Quads{ {Dir: quad.Predicate, Values: lookup("can")}, {Dir: quad.Object, Values: lookup("win")}, }, }, }, expect: "i", }, { message: "get a single quad linkage (internal)", add: quads1, query: "(\"i\" (:can $a))", shape: sh.Intersect{ lookup("i"), sh.NodesFrom{ Dir: quad.Subject, Quads: sh.Quads{ {Dir: quad.Predicate, Values: lookup("can")}, { Dir: quad.Object, Values: sh.Save{ Tags: []string{"$a"}, From: sh.AllNodes{}, }, }, }, }, }, expect: "i", }, { message: "tree constraint", add: []quad.Quad{ quad.Make("i", "like", "food", nil), quad.Make("food", "is", "good", nil), }, query: "(\"i\"\n" + "(:like\n" + "($a (:is :good))))", shape: sh.Intersect{ lookup("i"), sh.NodesFrom{ Dir: quad.Subject, Quads: sh.Quads{ {Dir: quad.Predicate, Values: lookup("like")}, { Dir: quad.Object, Values: sh.Save{ Tags: []string{"$a"}, From: sh.NodesFrom{ Dir: quad.Subject, Quads: sh.Quads{ {Dir: quad.Predicate, Values: lookup("is")}, {Dir: quad.Object, Values: lookup("good")}, }, }, }, }, }, }, }, expect: "i", tags: map[string]string{ "$a": "food", }, }, { message: "multiple constraint", add: []quad.Quad{ quad.Make("i", "like", "food", nil), quad.Make("i", "like", "beer", nil), quad.Make("you", "like", "beer", nil), }, query: `( $a (:like :beer) (:like "food") )`, shape: sh.Save{ Tags: []string{"$a"}, From: sh.Intersect{ sh.NodesFrom{ Dir: quad.Subject, Quads: sh.Quads{ {Dir: quad.Predicate, Values: lookup("like")}, {Dir: quad.Object, Values: lookup("beer")}, }, }, sh.NodesFrom{ Dir: quad.Subject, Quads: sh.Quads{ {Dir: quad.Predicate, Values: lookup("like")}, {Dir: quad.Object, Values: lookup("food")}, }, }, }, }, expect: "i", }, } func TestSexp(t *testing.T) { ctx := context.TODO() for _, test := range testQueries { t.Run(test.message, func(t *testing.T) { qs, _ := graph.NewQuadStore("memstore", "", nil) _ = testutil.MakeWriter(t, qs, nil, test.add...) s, _ := BuildShape(ctx, test.query) require.Equal(t, test.shape, s, "%s\n%#v\nvs\n%#v", test.message, test.shape, s) it := BuildIteratorTreeForQuery(ctx, qs, test.query).Iterate() if it.Next(ctx) != (test.expect != "") { t.Errorf("Failed to %s", test.message) } if test.expect != "" { qv, err := qs.ValueOf(quad.StringToValue(test.expect)) require.NoError(t, err) require.Equal(t, qv, it.Result()) tags := make(map[string]graph.Ref) it.TagResults(tags) for k, v := range test.tags { name, err := qs.NameOf(tags[k]) require.NoError(t, err) require.Equal(t, v, quad.ToString(name)) } if it.Next(ctx) { t.Error("too many results") } } }) } } ================================================ FILE: query/sexp/session.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package sexp // Defines a running session of the sexp query language. import ( "context" "errors" "fmt" "sort" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/query" ) const Name = "sexp" func init() { query.RegisterLanguage(query.Language{ Name: Name, Session: func(qs graph.QuadStore) query.Session { return NewSession(qs) }, }) } type Session struct { qs graph.QuadStore } func NewSession(qs graph.QuadStore) *Session { return &Session{qs: qs} } func (s *Session) Parse(input string) error { var parenDepth int for i, x := range input { if x == '(' { parenDepth++ } if x == ')' { parenDepth-- if parenDepth < 0 { min := 0 if (i - 10) > min { min = i - 10 } return fmt.Errorf("too many close parentheses at char %d: %s", i, input[min:i]) } } } if parenDepth > 0 { return query.ErrParseMore } if len(ParseString(input)) > 0 { return nil } return errors.New("invalid syntax") } func (s *Session) Execute(ctx context.Context, input string, opt query.Options) (query.Iterator, error) { switch opt.Collation { case query.Raw, query.REPL: default: return nil, &query.ErrUnsupportedCollation{Collation: opt.Collation} } it := BuildIteratorTreeForQuery(ctx, s.qs, input).Iterate() if err := it.Err(); err != nil { return nil, err } if opt.Limit > 0 { it = iterator.NewLimitNext(it, int64(opt.Limit)) } return &results{ s: s, col: opt.Collation, it: it, }, nil } type results struct { s *Session col query.Collation it iterator.Scanner nextPath bool err error } func (it *results) Next(ctx context.Context) bool { if it.nextPath && it.it.NextPath(ctx) { return true } it.nextPath = false if it.it.Next(ctx) { it.nextPath = true return true } return false } func (it *results) Result() interface{} { m := make(map[string]graph.Ref) it.it.TagResults(m) if it.col == query.Raw { return m } out := "****\n" tagKeys := make([]string, len(m)) i := 0 for k := range m { tagKeys[i] = k i++ } sort.Strings(tagKeys) for _, k := range tagKeys { if k == "$_" { continue } knv, err := it.s.qs.NameOf(m[k]) if err != nil { it.err = err return nil } out += fmt.Sprintf("%s : %s\n", k, knv) } return out } func (it *results) Err() error { return it.it.Err() } func (it *results) Close() error { return it.it.Close() } ================================================ FILE: query/shape/path.go ================================================ package shape import ( "context" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/quad" ) func IntersectShapes(s1, s2 Shape) Shape { switch s1 := s1.(type) { case AllNodes: return s2 case Intersect: if s2, ok := s2.(Intersect); ok { return append(s1, s2...) } return append(s1, s2) } return Intersect{s1, s2} } func IntersectOptional(s, opt Shape) Shape { var optional []Shape switch opt := opt.(type) { case Intersect: optional = []Shape(opt) case IntersectOpt: optional = make([]Shape, 0, len(opt.Sub)+len(opt.Opt)) optional = append(optional, opt.Sub...) optional = append(optional, opt.Opt...) default: optional = []Shape{opt} } if len(optional) == 0 { return s } switch s := s.(type) { case Intersect: return IntersectOpt{Sub: s, Opt: optional} case IntersectOpt: s.Opt = append(s.Opt, optional...) return s } return IntersectOpt{Sub: Intersect{s}, Opt: optional} } func UnionShapes(s1, s2 Shape) Union { if s1, ok := s1.(Union); ok { if s2, ok := s2.(Union); ok { return append(s1, s2...) } return append(s1, s2) } return Union{s1, s2} } func buildOut(from, via, labels Shape, tags []string, in bool) Shape { start, goal := quad.Subject, quad.Object if in { start, goal = goal, start } if len(tags) != 0 { via = Save{From: via, Tags: tags} } quads := make(Quads, 0, 3) if _, ok := from.(AllNodes); !ok { quads = append(quads, QuadFilter{ Dir: start, Values: from, }) } if _, ok := via.(AllNodes); !ok { quads = append(quads, QuadFilter{ Dir: quad.Predicate, Values: via, }) } if labels != nil { if _, ok := labels.(AllNodes); !ok { quads = append(quads, QuadFilter{ Dir: quad.Label, Values: labels, }) } } return NodesFrom{Quads: quads, Dir: goal} } func Out(from, via, labels Shape, tags ...string) Shape { return buildOut(from, via, labels, tags, false) } func In(from, via, labels Shape, tags ...string) Shape { return buildOut(from, via, labels, tags, true) } // InWithTags, OutWithTags, Both, BothWithTags func Predicates(from Shape, in bool) Shape { dir := quad.Subject if in { dir = quad.Object } return Unique{NodesFrom{ Quads: Quads{ {Dir: dir, Values: from}, }, Dir: quad.Predicate, }} } func SavePredicates(from Shape, in bool, tag string) Shape { preds := Save{ From: AllNodes{}, Tags: []string{tag}, } start := quad.Subject if in { start = quad.Object } var save Shape = NodesFrom{ Quads: Quads{ {Dir: quad.Predicate, Values: preds}, }, Dir: start, } return IntersectShapes(from, save) } func Labels(from Shape) Shape { return Unique{NodesFrom{ Quads: Union{ Quads{ {Dir: quad.Subject, Values: from}, }, Quads{ {Dir: quad.Object, Values: from}, }, }, Dir: quad.Label, }} } func SaveVia(from, via Shape, tag string, rev, opt bool) Shape { return SaveViaLabels(from, via, AllNodes{}, tag, rev, opt) } func SaveViaLabels(from, via, labels Shape, tag string, rev, opt bool) Shape { nodes := Save{ From: AllNodes{}, Tags: []string{tag}, } start, goal := quad.Subject, quad.Object if rev { start, goal = goal, start } quads := Quads{ {Dir: goal, Values: nodes}, {Dir: quad.Predicate, Values: via}, } if labels != nil { if _, ok := labels.(AllNodes); !ok { quads = append(quads, QuadFilter{ Dir: quad.Label, Values: labels, }) } } var save Shape = NodesFrom{ Quads: quads, Dir: start, } if opt { return IntersectOptional(from, save) } return IntersectShapes(from, save) } func Has(from, via, nodes Shape, rev bool) Shape { return HasLabels(from, via, AllNodes{}, nodes, rev) } func HasLabels(from, via, nodes, labels Shape, rev bool) Shape { start, goal := quad.Subject, quad.Object if rev { start, goal = goal, start } quads := make(Quads, 0, 3) if _, ok := nodes.(AllNodes); !ok { quads = append(quads, QuadFilter{ Dir: goal, Values: nodes, }) } if _, ok := via.(AllNodes); !ok { quads = append(quads, QuadFilter{ Dir: quad.Predicate, Values: via, }) } if labels != nil { if _, ok := labels.(AllNodes); !ok { quads = append(quads, QuadFilter{ Dir: quad.Label, Values: labels, }) } } if len(quads) == 0 { panic("empty has") } return IntersectShapes(from, NodesFrom{ Quads: quads, Dir: start, }) } func AddFilters(nodes Shape, filters ...ValueFilter) Shape { if len(filters) == 0 { return nodes } if s, ok := nodes.(Filter); ok { arr := make([]ValueFilter, 0, len(s.Filters)+len(filters)) arr = append(arr, s.Filters...) arr = append(arr, filters...) return Filter{From: s.From, Filters: arr} } if nodes == nil { nodes = AllNodes{} } return Filter{ From: nodes, Filters: filters, } } func Compare(nodes Shape, op iterator.Operator, v quad.Value) Shape { return AddFilters(nodes, Comparison{Op: op, Val: v}) } func Iterate(ctx context.Context, qs graph.QuadStore, s Shape) *iterator.Chain { it := BuildIterator(ctx, qs, s) return iterator.Iterate(ctx, it).On(qs) } ================================================ FILE: query/shape/shape.go ================================================ package shape import ( "context" "os" "reflect" "regexp" "strings" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" ) var ( debugShapes = os.Getenv("CAYLEY_DEBUG_SHAPES") == "true" debugOptimizer = os.Getenv("CAYLEY_DEBUG_OPTIMIZER") == "true" ) // Shape represent a query tree shape. type Shape interface { // BuildIterator constructs an iterator tree from a given shapes and binds it to QuadStore. BuildIterator(qs graph.QuadStore) iterator.Shape // Optimize runs an optimization pass over a query shape. // // It returns a bool that indicates if shape was replaced and should always return a copy of shape in this case. // In case no optimizations were made, it returns the same unmodified shape. // // If Optimizer is specified, it will be used instead of default optimizations. Optimize(ctx context.Context, r Optimizer) (Shape, bool) } type Optimizer interface { OptimizeShape(ctx context.Context, s Shape) (Shape, bool) } // Composite shape can be simplified to a tree of more basic shapes. type Composite interface { Simplify() Shape } // WalkFunc is used to visit all shapes in the tree. // If false is returned, branch will not be traversed further. type WalkFunc func(Shape) bool type resolveValues struct { qs graph.QuadStore } func (r resolveValues) OptimizeShape(ctx context.Context, s Shape) (Shape, bool) { if l, ok := s.(Lookup); ok { lv, err := l.resolve(r.qs) if err == nil { return lv, true } } return s, false } // Optimize applies generic optimizations for the tree. // If quad store is specified it will also resolve Lookups and apply any specific optimizations. // Should not be used with Simplify - it will fold query to a compact form again. func Optimize(ctx context.Context, s Shape, qs graph.QuadStore) (Shape, bool) { if s == nil { return nil, false } qs = graph.Unwrap(qs) var opt bool if qs != nil { // resolve all lookups earlier s, opt = s.Optimize(ctx, resolveValues{qs: qs}) } if s == nil { return Null{}, true } // generic optimizations var opt1 bool s, opt1 = s.Optimize(ctx, nil) if s == nil { return Null{}, true } opt = opt || opt1 // apply quadstore-specific optimizations if so, ok := qs.(Optimizer); ok && s != nil { var opt2 bool s, opt2 = s.Optimize(ctx, so) opt = opt || opt2 } if s == nil { return Null{}, true } return s, opt } var rtShape = reflect.TypeOf((*Shape)(nil)).Elem() // Walk calls provided function for each shape in the tree. func Walk(s Shape, fnc WalkFunc) { if s == nil { return } if !fnc(s) { return } walkReflect(reflect.ValueOf(s), fnc) } func walkReflect(rv reflect.Value, fnc WalkFunc) { rt := rv.Type() switch rv.Kind() { case reflect.Slice: if rt.Elem().ConvertibleTo(rtShape) { // all element are shapes - call function on each of them for i := 0; i < rv.Len(); i++ { Walk(rv.Index(i).Interface().(Shape), fnc) } } else { // elements are not shapes, but might contain them for i := 0; i < rv.Len(); i++ { walkReflect(rv.Index(i), fnc) } } case reflect.Map: keys := rv.MapKeys() if rt.Elem().ConvertibleTo(rtShape) { // all element are shapes - call function on each of them for _, k := range keys { Walk(rv.MapIndex(k).Interface().(Shape), fnc) } } else { // elements are not shapes, but might contain them for _, k := range keys { walkReflect(rv.MapIndex(k), fnc) } } case reflect.Struct: // visit all fields for i := 0; i < rt.NumField(); i++ { f := rt.Field(i) // if field is of shape type - call function on it // we skip anonymous fields because they were already visited as part of the parent if !f.Anonymous && f.Type.ConvertibleTo(rtShape) { Walk(rv.Field(i).Interface().(Shape), fnc) continue } // it might be a struct/map/slice field, so we need to go deeper walkReflect(rv.Field(i), fnc) } } } // InternalQuad is an internal representation of quad index in QuadStore. type InternalQuad struct { Subject refs.Ref Predicate refs.Ref Object refs.Ref Label refs.Ref } // Get returns a specified direction of the quad. func (q InternalQuad) Get(d quad.Direction) refs.Ref { switch d { case quad.Subject: return q.Subject case quad.Predicate: return q.Predicate case quad.Object: return q.Object case quad.Label: return q.Label default: return nil } } // Set assigns a specified direction of the quad to a given value. func (q *InternalQuad) Set(d quad.Direction, v refs.Ref) { switch d { case quad.Subject: q.Subject = v case quad.Predicate: q.Predicate = v case quad.Object: q.Object = v case quad.Label: q.Label = v default: panic(d) } } // QuadIndexer is an optional interface for quad stores that keep an index of quad directions. // // It is used to optimize shapes based on stats from these indexes. type QuadIndexer interface { // SizeOfIndex returns a size of a quad index with given constraints. SizeOfIndex(c map[quad.Direction]refs.Ref) (int64, bool) // LookupQuadIndex finds a quad that matches a given constraint. // It returns false if quad was not found, or there are multiple quads matching constraint. LookupQuadIndex(c map[quad.Direction]refs.Ref) (InternalQuad, bool) } // IsNull safely checks if shape represents an empty set. It accounts for both Null and nil. func IsNull(s Shape) bool { _, ok := s.(Null) return s == nil || ok } // BuildIterator optimizes the shape and builds a corresponding iterator tree. func BuildIterator(ctx context.Context, qs graph.QuadStore, s Shape) iterator.Shape { qs = graph.Unwrap(qs) if s != nil { if debugShapes || clog.V(2) { clog.Infof("shape: %#v", s) } s, _ = Optimize(ctx, s, qs) if debugOptimizer || clog.V(2) { clog.Infof("optimized: %#v", s) } } if IsNull(s) { return iterator.NewNull() } return s.BuildIterator(qs) } // Null represent an empty set. Mostly used as a safe alias for nil shape. type Null struct{} func (Null) BuildIterator(qs graph.QuadStore) iterator.Shape { return iterator.NewNull() } func (s Null) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if r != nil { return r.OptimizeShape(ctx, s) } return nil, true } // AllNodes represents all nodes in QuadStore. type AllNodes struct{} func (s AllNodes) BuildIterator(qs graph.QuadStore) iterator.Shape { return qs.NodesAllIterator() } func (s AllNodes) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if r != nil { return r.OptimizeShape(ctx, s) } return s, false } // Except excludes a set on nodes from a source. If source is nil, AllNodes is assumed. type Except struct { Exclude Shape // nodes to exclude From Shape // a set of all nodes to exclude from; nil means AllNodes } func (s Except) BuildIterator(qs graph.QuadStore) iterator.Shape { var all iterator.Shape if s.From != nil { all = s.From.BuildIterator(qs) } else { all = qs.NodesAllIterator() } if IsNull(s.Exclude) { return all } return iterator.NewNot(s.Exclude.BuildIterator(qs), all) } func (s Except) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { var opt bool s.Exclude, opt = s.Exclude.Optimize(ctx, r) if s.From != nil { var opta bool s.From, opta = s.From.Optimize(ctx, r) opt = opt || opta } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } if IsNull(s.Exclude) { return AllNodes{}, true } else if _, ok := s.Exclude.(AllNodes); ok { return nil, true } return s, opt } // ValueFilter is an interface for iterator wrappers that can filter node values. type ValueFilter interface { BuildIterator(qs graph.QuadStore, it iterator.Shape) iterator.Shape } // Filter filters all values from the source using a list of operations. type Filter struct { From Shape // source that will be filtered Filters []ValueFilter // filters to apply } func (s Filter) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.From) { return iterator.NewNull() } it := s.From.BuildIterator(qs) for _, f := range s.Filters { it = f.BuildIterator(qs, it) } return it } func (s Filter) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.From) { return nil, true } var opt bool s.From, opt = s.From.Optimize(ctx, r) if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } if IsNull(s.From) { return nil, true } else if len(s.Filters) == 0 { return s.From, true } return s, opt } var _ ValueFilter = Comparison{} // Comparison is a value filter that evaluates binary operation in reference to a fixed value. type Comparison struct { Op iterator.Operator Val quad.Value } func (f Comparison) BuildIterator(qs graph.QuadStore, it iterator.Shape) iterator.Shape { return iterator.NewComparison(it, f.Op, f.Val, qs) } var _ ValueFilter = Regexp{} // Regexp filters values using regular expression. // // Since regexp patterns can not be optimized in most cases, Wildcard should be used if possible. type Regexp struct { Re *regexp.Regexp Refs bool // allow to match IRIs } func (f Regexp) BuildIterator(qs graph.QuadStore, it iterator.Shape) iterator.Shape { if f.Refs { return iterator.NewRegexWithRefs(it, f.Re, qs) } return iterator.NewRegex(it, f.Re, qs) } var _ ValueFilter = Wildcard{} // Wildcard is a filter for string patterns. // // % - zero or more characters // ? - exactly one character type Wildcard struct { Pattern string // allowed wildcards are: % and ? } // Regexp returns an analog regexp pattern in format accepted by Go stdlib (RE2). func (f Wildcard) Regexp() string { const any = `%` // escape all meta-characters in pattern string pattern := regexp.QuoteMeta(f.Pattern) // if the pattern is anchored, add regexp analog for it if !strings.HasPrefix(pattern, any) { pattern = "^" + pattern } else { pattern = strings.TrimPrefix(pattern, any) } if !strings.HasSuffix(pattern, any) { pattern = pattern + "$" } else { pattern = strings.TrimSuffix(pattern, any) } // replace wildcards pattern = strings.NewReplacer( any, `.*`, `\?`, `.`, ).Replace(pattern) return pattern } func (f Wildcard) BuildIterator(qs graph.QuadStore, it iterator.Shape) iterator.Shape { if f.Pattern == "" { return iterator.NewNull() } else if strings.Trim(f.Pattern, "%") == "" { return it } re, err := regexp.Compile(f.Regexp()) if err != nil { return iterator.NewError(err) } return iterator.NewRegexWithRefs(it, re, qs) } // Count returns a count of objects in source as a single value. It always returns exactly one value. type Count struct { Values Shape } func (s Count) BuildIterator(qs graph.QuadStore) iterator.Shape { var it iterator.Shape if IsNull(s.Values) { it = iterator.NewNull() } else { it = s.Values.BuildIterator(qs) } return iterator.NewCount(it, qs) } func (s Count) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.Values) { return Fixed{refs.PreFetched(quad.Int(0))}, true } var opt bool s.Values, opt = s.Values.Optimize(ctx, r) if IsNull(s.Values) { return Fixed{refs.PreFetched(quad.Int(0))}, true } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } // TODO: ask QS to estimate size - if it exact, then we can use it return s, opt } // QuadFilter is a constraint used to filter quads that have a certain set of values on a given direction. // Analog of LinksTo iterator. type QuadFilter struct { Dir quad.Direction Values Shape } // buildIterator is not exposed to force to use Quads and group filters together. func (s QuadFilter) buildIterator(qs graph.QuadStore) iterator.Shape { if s.Values == nil { return iterator.NewNull() } else if v, ok := One(s.Values); ok { return qs.QuadIterator(s.Dir, v) } if s.Dir == quad.Any { panic("direction is not set") } sub := s.Values.BuildIterator(qs) return graph.NewLinksTo(qs, sub, s.Dir) } // Quads is a selector of quads with a given set of node constraints. Empty or nil Quads is equivalent to AllQuads. // Equivalent to And(AllQuads,LinksTo*) iterator tree. type Quads []QuadFilter func (s *Quads) Intersect(q ...QuadFilter) { *s = append(*s, q...) } func (s Quads) BuildIterator(qs graph.QuadStore) iterator.Shape { if len(s) == 0 { return qs.QuadsAllIterator() } its := make([]iterator.Shape, 0, len(s)) for _, f := range s { its = append(its, f.buildIterator(qs)) } if len(its) == 1 { return its[0] } return iterator.NewAnd(its...) } func (s Quads) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { var opt bool sw := 0 realloc := func() { if !opt { opt = true nq := make(Quads, len(s)) copy(nq, s) s = nq } } // TODO: multiple constraints on the same dir -> merge as Intersect on Values of this dir for i := 0; i < len(s); i++ { f := s[i] if f.Values == nil { return nil, true } v, ok := f.Values.Optimize(ctx, r) if v == nil { return nil, true } if ok { realloc() s[i].Values = v } switch s[i].Values.(type) { case Fixed: realloc() s[sw], s[i] = s[i], s[sw] sw++ } } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } return s, opt } // NodesFrom extracts nodes on a given direction from source quads. Similar to HasA iterator. type NodesFrom struct { Dir quad.Direction Quads Shape } func (s NodesFrom) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.Quads) { return iterator.NewNull() } sub := s.Quads.BuildIterator(qs) if s.Dir == quad.Any { panic("direction is not set") } return graph.NewHasA(qs, sub, s.Dir) } func (s NodesFrom) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.Quads) { return nil, true } var opt bool s.Quads, opt = s.Quads.Optimize(ctx, r) if r != nil { // ignore default optimizations ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } q, ok := s.Quads.(Quads) if !ok { return s, opt } // HasA(x, LinksTo(x, y)) == y if len(q) == 1 && q[0].Dir == s.Dir { return q[0].Values, true } // collect all fixed tags and push them up the tree var ( tags map[string]refs.Ref nquad Quads ) for i, f := range q { if ft, ok := f.Values.(FixedTags); ok { if tags == nil { // allocate map and clone quad filters tags = make(map[string]refs.Ref) nquad = make([]QuadFilter, len(q)) copy(nquad, q) q = nquad } q[i].Values = ft.On for k, v := range ft.Tags { tags[k] = v } } } if tags != nil { // re-run optimization without fixed tags ns, _ := NodesFrom{Dir: s.Dir, Quads: q}.Optimize(ctx, r) return FixedTags{On: ns, Tags: tags}, true } var ( // if quad filter contains one fixed value, it will be added to the map filt map[quad.Direction]refs.Ref // if we see a Save from AllNodes, we will write it here, since it's a Save on quad direction save map[quad.Direction][]string // how many filters are recognized n int ) for _, f := range q { if v, ok := One(f.Values); ok { if filt == nil { filt = make(map[quad.Direction]refs.Ref) } if _, ok := filt[f.Dir]; ok { return s, opt // just to be safe } filt[f.Dir] = v n++ } else if sv, ok := f.Values.(Save); ok { if _, ok = sv.From.(AllNodes); ok { if save == nil { save = make(map[quad.Direction][]string) } save[f.Dir] = append(save[f.Dir], sv.Tags...) n++ } } } if n == len(q) { // if all filters were recognized we can merge this tree as a single iterator with multiple // constraints and multiple save commands over the same set of quads ns, _ := QuadsAction{ Result: s.Dir, // this is still a HasA, remember? Filter: filt, Save: save, }.Optimize(ctx, r) return ns, true } // TODO return s, opt } var _ Composite = QuadsAction{} // QuadsAction represents a set of actions that can be done to a set of quads in a single scan pass. // It filters quads according to Filter constraints (equivalent of LinksTo), tags directions using tags in Save field // and returns a specified quad direction as result of the iterator (equivalent of HasA). // Optionally, Size field may be set to indicate an approximate number of quads that will be returned by this query. type QuadsAction struct { Size int64 // approximate size; zero means undefined Result quad.Direction Save map[quad.Direction][]string Filter map[quad.Direction]refs.Ref } func (s *QuadsAction) SetFilter(d quad.Direction, v refs.Ref) { if s.Filter == nil { s.Filter = make(map[quad.Direction]refs.Ref) } s.Filter[d] = v } func (s QuadsAction) Clone() QuadsAction { if n := len(s.Save); n != 0 { s2 := make(map[quad.Direction][]string, n) for k, v := range s.Save { s2[k] = v } s.Save = s2 } else { s.Save = nil } if n := len(s.Filter); n != 0 { f2 := make(map[quad.Direction]refs.Ref, n) for k, v := range s.Filter { f2[k] = v } s.Filter = f2 } else { s.Filter = nil } return s } func (s QuadsAction) simplify() NodesFrom { q := make(Quads, 0, len(s.Save)+len(s.Filter)) for dir, val := range s.Filter { q = append(q, QuadFilter{Dir: dir, Values: Fixed{val}}) } for dir, tags := range s.Save { q = append(q, QuadFilter{Dir: dir, Values: Save{From: AllNodes{}, Tags: tags}}) } return NodesFrom{Dir: s.Result, Quads: q} } func (s QuadsAction) SimplifyFrom(quads Shape) Shape { q := make(Quads, 0, len(s.Save)) for dir, tags := range s.Save { q = append(q, QuadFilter{Dir: dir, Values: Save{From: AllNodes{}, Tags: tags}}) } if len(q) != 0 { quads = IntersectShapes(quads, q) } return NodesFrom{Dir: s.Result, Quads: quads} } func (s QuadsAction) Simplify() Shape { return s.simplify() } func (s QuadsAction) BuildIterator(qs graph.QuadStore) iterator.Shape { h := s.simplify() return h.BuildIterator(qs) } func (s QuadsAction) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if r != nil { return r.OptimizeShape(ctx, s) } // if optimizer has stats for quad indexes we can use them to do more ind, ok := r.(QuadIndexer) if !ok { return s, false } if s.Size > 0 { // already optimized; specific for QuadIndexer optimization return s, false } sz, exact := ind.SizeOfIndex(s.Filter) if !exact { return s, false } s.Size = sz // computing size is already an optimization if sz == 0 { // nothing here, collapse the tree return nil, true } else if sz == 1 { // only one quad matches this set of filters // try to load it from quad store, do all operations and bake result as a fixed node/tags if q, ok := ind.LookupQuadIndex(s.Filter); ok { fx := Fixed{q.Get(s.Result)} if len(s.Save) == 0 { return fx, true } ft := FixedTags{On: fx, Tags: make(map[string]refs.Ref)} for d, tags := range s.Save { for _, t := range tags { ft.Tags[t] = q.Get(d) } } return ft, true } } if sz < int64(MaterializeThreshold) { // if this set is small enough - materialize it return Materialize{Values: s, Size: int(sz)}, true } return s, true } // One checks if Shape represents a single fixed value and returns it. func One(s Shape) (refs.Ref, bool) { switch s := s.(type) { case Fixed: if len(s) == 1 { return s[0], true } } return nil, false } // Fixed is a static set of nodes. Defined only for a particular QuadStore. type Fixed []refs.Ref func (s *Fixed) Add(v ...refs.Ref) { *s = append(*s, v...) } func (s Fixed) BuildIterator(qs graph.QuadStore) iterator.Shape { it := iterator.NewFixed() for _, v := range s { if _, ok := v.(quad.Value); ok { panic("quad value in fixed iterator") } it.Add(v) } return it } func (s Fixed) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if len(s) == 0 { return nil, true } if r != nil { return r.OptimizeShape(ctx, s) } return s, false } // FixedTags adds a set of fixed tag values to query results. It does not affect query execution in any other way. // // Shape implementations should try to push these objects up the tree during optimization process. type FixedTags struct { Tags map[string]refs.Ref On Shape } func (s FixedTags) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.On) { return iterator.NewNull() } it := s.On.BuildIterator(qs) sv := iterator.NewSave(it) for k, v := range s.Tags { sv.AddFixedTag(k, v) } return sv } func (s FixedTags) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.On) { return nil, true } var opt bool s.On, opt = s.On.Optimize(ctx, r) if len(s.Tags) == 0 { return s.On, true } else if s2, ok := s.On.(FixedTags); ok { tags := make(map[string]refs.Ref, len(s.Tags)+len(s2.Tags)) for k, v := range s.Tags { tags[k] = v } for k, v := range s2.Tags { tags[k] = v } s, opt = FixedTags{On: s2.On, Tags: tags}, true } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } return s, opt } // Lookup is a static set of values that must be resolved to nodes by QuadStore. type Lookup []quad.Value func (s *Lookup) Add(v ...quad.Value) { *s = append(*s, v...) } var _ valueResolver = graph.QuadStore(nil) type valueResolver interface { ValueOf(v quad.Value) (refs.Ref, error) } func (s Lookup) resolve(qs valueResolver) (Shape, error) { // TODO: check if QS supports batch lookup vals := make([]refs.Ref, 0, len(s)) for _, v := range s { gv, err := qs.ValueOf(v) if err != nil { return nil, err } if gv != nil { vals = append(vals, gv) } } if len(vals) == 0 { return nil, nil } return Fixed(vals), nil } func (s Lookup) BuildIterator(qs graph.QuadStore) iterator.Shape { f, err := s.resolve(qs) if err != nil { return iterator.NewError(err) } if IsNull(f) { return iterator.NewNull() } return f.BuildIterator(qs) } func (s Lookup) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if r == nil { return s, false } ns, opt := r.OptimizeShape(ctx, s) if opt { return ns, true } if qs, ok := r.(valueResolver); ok { res, err := s.resolve(qs) if err == nil { ns, opt = res, true } } return ns, opt } var MaterializeThreshold = 100 // TODO: tune // Materialize loads results of sub-query into memory during execution to speedup iteration. type Materialize struct { Size int // approximate size; zero means undefined Values Shape } func (s Materialize) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.Values) { return iterator.NewNull() } it := s.Values.BuildIterator(qs) return iterator.NewMaterializeWithSize(it, int64(s.Size)) } func (s Materialize) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.Values) { return nil, true } var opt bool s.Values, opt = s.Values.Optimize(ctx, r) if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } return s, opt } func clearFixedTags(arr []Shape) ([]Shape, map[string]refs.Ref) { var tags map[string]refs.Ref for i := 0; i < len(arr); i++ { if ft, ok := arr[i].(FixedTags); ok { if tags == nil { tags = make(map[string]refs.Ref) na := make([]Shape, len(arr)) copy(na, arr) arr = na } arr[i] = ft.On for k, v := range ft.Tags { tags[k] = v } } } return arr, tags } // Intersect computes an intersection of nodes between multiple queries. Similar to And iterator. type Intersect []Shape func (s Intersect) BuildIterator(qs graph.QuadStore) iterator.Shape { if len(s) == 0 { return iterator.NewNull() } sub := make([]iterator.Shape, 0, len(s)) for _, c := range s { sub = append(sub, c.BuildIterator(qs)) } if len(sub) == 1 { return sub[0] } return iterator.NewAnd(sub...) } func (s Intersect) Optimize(ctx context.Context, r Optimizer) (sout Shape, opt bool) { if len(s) == 0 { return nil, true } // function to lazily reallocate a copy of Intersect slice realloc := func() { if !opt { arr := make(Intersect, len(s)) copy(arr, s) s = arr } } // optimize sub-iterators, return empty set if Null is found for i := 0; i < len(s); i++ { c := s[i] if IsNull(c) { return nil, true } v, ok := c.Optimize(ctx, r) if !ok { continue } realloc() opt = true if IsNull(v) { return nil, true } s[i] = v } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } if arr, ft := clearFixedTags([]Shape(s)); ft != nil { ns, _ := FixedTags{On: Intersect(arr), Tags: ft}.Optimize(ctx, r) return ns, true } var ( onlyAll = true // contains only AllNodes shapes hasAll = false fixed []Fixed // we will collect all Fixed, and will place it as a first iterator tags []string // if we find a Save inside, we will push it outside of Intersect quads Quads // also, collect all quad filters into a single set optional []Shape ) remove := func(i *int, optimized bool) { realloc() if optimized { opt = true } v := *i s = append(s[:v], s[v+1:]...) v-- *i = v } // second pass - remove AllNodes, merge Quads, collect Fixed, collect Save, merge Intersects for i := 0; i < len(s); i++ { c := s[i] switch c := c.(type) { case AllNodes: // remove AllNodes - it's useless in the intersection remove(&i, true) hasAll = true continue // prevent resetting of onlyAll case Quads: // merge all quad filters remove(&i, false) if quads == nil { quads = c[:len(c):len(c)] } else { opt = true quads = append(quads, c...) } case Fixed: // collect all Fixed sets remove(&i, true) fixed = append(fixed, c) case Intersect: // merge with other Intersects remove(&i, true) s = append(s, c...) case IntersectOpt: // merge with IntersectOpt remove(&i, true) s = append(s, c.Sub...) optional = append(optional, c.Opt...) case Save: // push Save outside of Intersect realloc() opt = true tags = append(tags, c.Tags...) s[i] = c.From i-- } onlyAll = false } if onlyAll { return AllNodes{}, true } if len(tags) != 0 { // don't forget to move Save outside of Intersect at the end defer func() { if IsNull(sout) { return } sv := Save{From: sout, Tags: tags} var topt bool sout, topt = sv.Optimize(ctx, r) opt = opt || topt }() } if len(optional) != 0 { // don't forget to add optional paths defer func() { if IsNull(sout) { return } out := IntersectOpt{Opt: optional} if so, ok := sout.(Intersect); ok { out.Sub = so } else { out.Sub = Intersect{sout} } var topt bool sout, topt = out.Optimize(ctx, r) opt = opt || topt }() } if quads != nil { nq, qopt := quads.Optimize(ctx, r) if IsNull(nq) { return nil, true } opt = opt || qopt s = append(s, nq) } // TODO: intersect fixed if len(fixed) == 1 { fix := fixed[0] if len(s) == 1 { // try to push fixed down the tree switch sf := s[0].(type) { case QuadsAction: // TODO: accept an array of Fixed values if len(fix) == 1 { // we have a single value in Fixed that is intersected with HasA tree // this means we can add a new constraint: LinksTo(HasA.Dir, fixed) // result direction of HasA will be preserved fv := fix[0] if v := sf.Filter[sf.Result]; v != nil { // we have the same direction set as a fixed constraint - do filtering if refs.ToKey(v) != refs.ToKey(fv) { return nil, true } return sf, true } sf = sf.Clone() sf.SetFilter(sf.Result, fv) // LinksTo(HasA.Dir, fixed) sf.Size = 0 // re-calculate size ns, _ := sf.Optimize(ctx, r) return ns, true } case NodesFrom: if sq, ok := sf.Quads.(Quads); ok { // an optimization above is valid for NodesFrom+Quads as well // we can add the same constraint to Quads and remove Fixed qi := -1 for i, qf := range sq { if qf.Dir == sf.Dir { qi = i break } } if qi < 0 { // no filter on this direction - append sf.Quads = append(Quads{ {Dir: sf.Dir, Values: fix}, }, sq...) } else { // already have a filter on this direction - push Fixed inside it sq = append(Quads{}, sq...) sf.Quads = sq qf := &sq[qi] qf.Values = IntersectShapes(fix, qf.Values) } return sf, true } } } // place fixed as a first iterator s = append(s, nil) copy(s[1:], s) s[0] = fix } else if len(fixed) > 1 { ns := make(Intersect, len(s)+len(fixed)) for i, f := range fixed { ns[i] = f } copy(ns[len(fixed):], s) s = ns } if len(s) == 0 { if hasAll { return AllNodes{}, true } return nil, true } else if len(s) == 1 { return s[0], true } // TODO: optimize order return s, opt } // IntersectOpt is like Intersect but it also joins optional query shapes to the main query. type IntersectOpt struct { Sub Intersect Opt []Shape } func (s *IntersectOpt) Add(arr ...Shape) { s.Sub = append(s.Sub, arr...) } func (s *IntersectOpt) AddOptional(arr ...Shape) { s.Opt = append(s.Opt, arr...) } func (s IntersectOpt) BuildIterator(qs graph.QuadStore) iterator.Shape { if len(s.Sub) == 0 && len(s.Opt) == 0 { return iterator.NewNull() } if len(s.Sub) == 0 { if len(s.Opt) == 0 { return iterator.NewNull() } s.Sub = Intersect{AllNodes{}} } sub := make([]iterator.Shape, 0, len(s.Sub)) opt := make([]iterator.Shape, 0, len(s.Opt)) for _, c := range s.Sub { sub = append(sub, c.BuildIterator(qs)) } for _, c := range s.Opt { opt = append(opt, c.BuildIterator(qs)) } if len(sub) == 1 && len(opt) == 0 { return sub[0] } it := iterator.NewAnd(sub...) for _, sit := range opt { it.AddOptionalIterator(sit) } return it } func (s IntersectOpt) Optimize(ctx context.Context, r Optimizer) (_ Shape, opt bool) { // optimize optional shapes first, reallocate if necessary newSlice := false realloc := func() { opt = true if newSlice { return } newSlice = true s.Opt = append([]Shape{}, s.Opt...) } for i := 0; i < len(s.Opt); i++ { o := s.Opt[i] if IsNull(o) { realloc() s.Opt = append(s.Opt[:i], s.Opt[i+1:]...) i-- continue } o, opt2 := o.Optimize(ctx, r) if !opt2 { continue } realloc() if IsNull(o) { s.Opt = append(s.Opt[:i], s.Opt[i+1:]...) i-- } else { s.Opt[i] = o } } if len(s.Opt) == 0 { // no optional - replace with a regular intersection si, _ := s.Sub.Optimize(ctx, r) return si, true } if len(s.Sub) == 0 { // force at least All to be in the intersection s.Sub = Intersect{AllNodes{}} opt = true } else { sub, opt2 := s.Sub.Optimize(ctx, r) if IsNull(sub) { return nil, true } opt = opt || opt2 switch sub := sub.(type) { case Intersect: s.Sub = sub case IntersectOpt: s = sub opt = true default: s.Sub = Intersect{sub} opt = true } } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } return s, opt } // Union joins results of multiple queries together. It does not make results unique. type Union []Shape func (s Union) BuildIterator(qs graph.QuadStore) iterator.Shape { if len(s) == 0 { return iterator.NewNull() } sub := make([]iterator.Shape, 0, len(s)) for _, c := range s { sub = append(sub, c.BuildIterator(qs)) } if len(sub) == 1 { return sub[0] } return iterator.NewOr(sub...) } func (s Union) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { var opt bool realloc := func() { if !opt { arr := make(Union, len(s)) copy(arr, s) s = arr } } // optimize subiterators for i := 0; i < len(s); i++ { c := s[i] if c == nil { continue } v, ok := c.Optimize(ctx, r) if !ok { continue } realloc() opt = true s[i] = v } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } if arr, ft := clearFixedTags([]Shape(s)); ft != nil { ns, _ := FixedTags{On: Union(arr), Tags: ft}.Optimize(ctx, r) return ns, true } // second pass - remove Null for i := 0; i < len(s); i++ { c := s[i] if IsNull(c) { realloc() opt = true s = append(s[:i], s[i+1:]...) } } if len(s) == 0 { return nil, true } else if len(s) == 1 { return s[0], true } // TODO: join Fixed return s, opt } // Page provides a simple form of pagination. Can be used to skip or limit results. type Page struct { From Shape Skip int64 Limit int64 // zero means unlimited } func (s Page) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.From) { return iterator.NewNull() } it := s.From.BuildIterator(qs) if s.Skip > 0 { it = iterator.NewSkip(it, s.Skip) } if s.Limit > 0 { it = iterator.NewLimit(it, s.Limit) } return it } func (s Page) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.From) { return nil, true } var opt bool s.From, opt = s.From.Optimize(ctx, r) if s.Skip <= 0 && s.Limit <= 0 { return s.From, true } if p, ok := s.From.(Page); ok { p2 := p.ApplyPage(s) if p2 == nil { return nil, true } s, opt = *p2, true } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } // TODO: check size return s, opt } func (s Page) ApplyPage(p Page) *Page { s.Skip += p.Skip if s.Limit > 0 { s.Limit -= p.Skip if s.Limit <= 0 { return nil } if p.Limit > 0 && s.Limit > p.Limit { s.Limit = p.Limit } } else { s.Limit = p.Limit } return &s } // Unique makes query results unique. type Unique struct { From Shape } func (s Unique) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.From) { return iterator.NewNull() } it := s.From.BuildIterator(qs) return iterator.NewUnique(it) } func (s Unique) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.From) { return nil, true } var opt bool s.From, opt = s.From.Optimize(ctx, r) if IsNull(s.From) { return nil, true } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } return s, opt } // Save tags a results of query with provided tags. type Save struct { Tags []string From Shape } func (s Save) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.From) { return iterator.NewNull() } it := s.From.BuildIterator(qs) if len(s.Tags) != 0 { return iterator.NewSave(it, s.Tags...) } return it } func (s Save) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.From) { return nil, true } var opt bool s.From, opt = s.From.Optimize(ctx, r) if len(s.Tags) == 0 { return s.From, true } else if IsNull(s.From) { return nil, true } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } return s, opt } func FilterQuads(subject, predicate, object, label []quad.Value) Shape { var q Quads if len(subject) != 0 { q = append(q, QuadFilter{Dir: quad.Subject, Values: Lookup(subject)}) } if len(predicate) != 0 { q = append(q, QuadFilter{Dir: quad.Predicate, Values: Lookup(predicate)}) } if len(object) != 0 { q = append(q, QuadFilter{Dir: quad.Object, Values: Lookup(object)}) } if len(label) != 0 { q = append(q, QuadFilter{Dir: quad.Label, Values: Lookup(label)}) } return q } type Sort struct { From Shape } func (s Sort) BuildIterator(qs graph.QuadStore) iterator.Shape { if IsNull(s.From) { return iterator.NewNull() } it := s.From.BuildIterator(qs) return iterator.NewSort(qs, it) } func (s Sort) Optimize(ctx context.Context, r Optimizer) (Shape, bool) { if IsNull(s.From) { return nil, true } var opt bool s.From, opt = s.From.Optimize(ctx, r) if IsNull(s.From) { return nil, true } if r != nil { ns, nopt := r.OptimizeShape(ctx, s) return ns, opt || nopt } return s, opt } ================================================ FILE: query/shape/shape_test.go ================================================ // Copyright 2014 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package shape_test import ( "context" "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/graphmock" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/refs" . "github.com/cayleygraph/cayley/query/shape" "github.com/cayleygraph/quad" ) func intVal(v int) refs.Ref { return graphmock.IntVal(v) } var _ Optimizer = ValLookup(nil) var _ graph.QuadStore = ValLookup(nil) type ValLookup map[quad.Value]refs.Ref func (qs ValLookup) OptimizeShape(ctx context.Context, s Shape) (Shape, bool) { return s, false // emulate dumb quad store } func (qs ValLookup) ValueOf(v quad.Value) (refs.Ref, error) { return qs[v], nil } func (ValLookup) NewQuadWriter() (quad.WriteCloser, error) { panic("not implemented") } func (ValLookup) ApplyDeltas(_ []graph.Delta, _ graph.IgnoreOpts) error { panic("not implemented") } func (ValLookup) Quad(_ refs.Ref) (quad.Quad, error) { panic("not implemented") } func (ValLookup) QuadIterator(_ quad.Direction, _ refs.Ref) iterator.Shape { panic("not implemented") } func (ValLookup) QuadIteratorSize(ctx context.Context, d quad.Direction, val refs.Ref) (refs.Size, error) { panic("not implemented") } func (ValLookup) NodesAllIterator() iterator.Shape { panic("not implemented") } func (ValLookup) QuadsAllIterator() iterator.Shape { panic("not implemented") } func (ValLookup) NameOf(_ refs.Ref) (quad.Value, error) { panic("not implemented") } func (ValLookup) Stats(ctx context.Context, exact bool) (graph.Stats, error) { panic("not implemented") } func (ValLookup) Close() error { panic("not implemented") } func (ValLookup) QuadDirection(_ refs.Ref, _ quad.Direction) (refs.Ref, error) { panic("not implemented") } func (ValLookup) Type() string { panic("not implemented") } func emptySet() Shape { return NodesFrom{ Dir: quad.Predicate, Quads: Intersect{Quads{ {Dir: quad.Object, Values: Lookup{quad.IRI("not-existent")}, }, }}, } } var optimizeCases = []struct { name string from Shape expect Shape opt bool qs ValLookup }{ { name: "all", from: AllNodes{}, opt: false, expect: AllNodes{}, }, { name: "page min limit", from: Page{ Limit: 5, From: Page{ Limit: 3, From: AllNodes{}, }, }, opt: true, expect: Page{ Limit: 3, From: AllNodes{}, }, }, { name: "page skip and limit", from: Page{ Skip: 3, Limit: 3, From: Page{ Skip: 2, Limit: 5, From: AllNodes{}, }, }, opt: true, expect: Page{ Skip: 5, Limit: 2, From: AllNodes{}, }, }, { name: "intersect tagged all", from: Intersect{Save{Tags: []string{"id"}, From: AllNodes{}}}, opt: true, expect: Save{Tags: []string{"id"}, From: AllNodes{}}, }, { name: "intersect quads and lookup resolution", from: Intersect{ Quads{ {Dir: quad.Subject, Values: Lookup{quad.IRI("bob")}}, }, Quads{ {Dir: quad.Object, Values: Lookup{quad.IRI("alice")}}, }, }, opt: true, expect: Quads{ {Dir: quad.Subject, Values: Fixed{intVal(1)}}, {Dir: quad.Object, Values: Fixed{intVal(2)}}, }, qs: ValLookup{ quad.IRI("bob"): intVal(1), quad.IRI("alice"): intVal(2), }, }, { name: "intersect nodes, remove all, join intersects", from: Intersect{ AllNodes{}, NodesFrom{Dir: quad.Subject, Quads: Quads{}}, Intersect{ Lookup{quad.IRI("alice")}, Unique{NodesFrom{Dir: quad.Object, Quads: Quads{}}}, }, }, opt: true, expect: Intersect{ Fixed{intVal(1)}, QuadsAction{Result: quad.Subject}, Unique{QuadsAction{Result: quad.Object}}, }, qs: ValLookup{ quad.IRI("alice"): intVal(1), }, }, { name: "push Save out of intersect", from: Intersect{ Save{ Tags: []string{"id"}, From: NodesFrom{Dir: quad.Subject, Quads: Quads{}}, }, Unique{NodesFrom{Dir: quad.Object, Quads: Quads{}}}, }, opt: true, expect: Save{ Tags: []string{"id"}, From: Intersect{ QuadsAction{Result: quad.Subject}, Unique{QuadsAction{Result: quad.Object}}, }, }, }, { name: "collapse empty set", from: Intersect{Quads{ {Dir: quad.Subject, Values: Union{ Unique{emptySet()}, }}, }}, opt: true, expect: Null{}, }, { // remove "all nodes" in intersect, merge Fixed and order them first name: "remove all in intersect and reorder", from: Intersect{ AllNodes{}, Fixed{intVal(1), intVal(2)}, Save{From: AllNodes{}, Tags: []string{"all"}}, Fixed{intVal(2)}, }, opt: true, expect: Save{ From: Intersect{ Fixed{intVal(1), intVal(2)}, Fixed{intVal(2)}, }, Tags: []string{"all"}, }, }, { name: "remove HasA-LinksTo pairs", from: NodesFrom{ Dir: quad.Subject, Quads: Quads{{ Dir: quad.Subject, Values: Fixed{intVal(1)}, }}, }, opt: true, expect: Fixed{intVal(1)}, }, { // pop fixed tags to the top of the tree name: "pop fixed tags", from: NodesFrom{Dir: quad.Subject, Quads: Quads{ QuadFilter{Dir: quad.Predicate, Values: Intersect{ FixedTags{ Tags: map[string]refs.Ref{"foo": intVal(1)}, On: NodesFrom{Dir: quad.Subject, Quads: Quads{ QuadFilter{Dir: quad.Object, Values: FixedTags{ Tags: map[string]refs.Ref{"bar": intVal(2)}, On: Fixed{intVal(3)}, }}, }, }, }, }}, }}, opt: true, expect: FixedTags{ Tags: map[string]refs.Ref{"foo": intVal(1), "bar": intVal(2)}, On: NodesFrom{Dir: quad.Subject, Quads: Quads{ QuadFilter{Dir: quad.Predicate, Values: QuadsAction{ Result: quad.Subject, Filter: map[quad.Direction]refs.Ref{quad.Object: intVal(3)}, }}, }}, }, }, { // remove optional empty set from intersect name: "remove optional empty set", from: IntersectOpt{ Sub: Intersect{ AllNodes{}, Save{From: AllNodes{}, Tags: []string{"all"}}, Fixed{intVal(2)}, }, Opt: []Shape{Save{ From: emptySet(), Tags: []string{"name"}, }}, }, opt: true, expect: Save{ From: Fixed{intVal(2)}, Tags: []string{"all"}, }, }, { // push fixed node from intersect into nodes.quads name: "push fixed into nodes.quads", from: Intersect{ Fixed{intVal(1)}, NodesFrom{ Dir: quad.Subject, Quads: Quads{ {Dir: quad.Predicate, Values: Fixed{intVal(2)}}, { Dir: quad.Object, Values: NodesFrom{ Dir: quad.Subject, Quads: Quads{ {Dir: quad.Predicate, Values: Fixed{intVal(2)}}, }, }, }, }, }, }, opt: true, expect: NodesFrom{ Dir: quad.Subject, Quads: Quads{ {Dir: quad.Subject, Values: Fixed{intVal(1)}}, {Dir: quad.Predicate, Values: Fixed{intVal(2)}}, { Dir: quad.Object, Values: QuadsAction{ Result: quad.Subject, Filter: map[quad.Direction]refs.Ref{ quad.Predicate: intVal(2), }, }, }, }, }, }, { name: "all optional", from: Intersect{IntersectOpt{ Sub: Intersect{ Save{Tags: []string{"id"}, From: AllNodes{}}, }, Opt: []Shape{ NodesFrom{Dir: quad.Subject, Quads: Quads{ QuadFilter{Dir: quad.Object, Values: Save{Tags: []string{"status"}, From: AllNodes{}}}, QuadFilter{Dir: quad.Predicate, Values: Fixed{intVal(1)}}, }}, }, }}, opt: true, expect: Save{ Tags: []string{"id"}, From: IntersectOpt{ Sub: Intersect{AllNodes{}}, Opt: []Shape{ QuadsAction{Result: quad.Subject, Save: map[quad.Direction][]string{quad.Object: {"status"}}, Filter: map[quad.Direction]refs.Ref{quad.Predicate: intVal(1)}, }, }, }, }, }, } func TestOptimize(t *testing.T) { for _, c := range optimizeCases { t.Run(c.name, func(t *testing.T) { qs := c.qs got, opt := Optimize(context.TODO(), c.from, qs) assert.Equal(t, c.expect, got) assert.Equal(t, c.opt, opt) }) } } func TestWalk(t *testing.T) { var s Shape = NodesFrom{ Dir: quad.Subject, Quads: Quads{ {Dir: quad.Subject, Values: Fixed{intVal(1)}}, {Dir: quad.Predicate, Values: Fixed{intVal(2)}}, { Dir: quad.Object, Values: QuadsAction{ Result: quad.Subject, Filter: map[quad.Direction]refs.Ref{ quad.Predicate: intVal(2), }, }, }, }, } var types []string Walk(s, func(s Shape) bool { types = append(types, reflect.TypeOf(s).String()) return true }) require.Equal(t, []string{ "shape.NodesFrom", "shape.Quads", "shape.Fixed", "shape.Fixed", "shape.QuadsAction", }, types) } ================================================ FILE: schema/loader.go ================================================ package schema import ( "context" "errors" "fmt" "reflect" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/iterator" ) var ( errNotFound = errors.New("not found") errRequiredFieldIsMissing = errors.New("required field is missing") ) // Optimize flags controls an optimization step performed before queries. var Optimize = true // IsNotFound check if error is related to a missing object (either because of wrong ID or because of type constrains). func IsNotFound(err error) bool { return err == errNotFound || err == errRequiredFieldIsMissing } // LoadTo will load a sub-graph of objects starting from ids (or from any nodes, if empty) // to a destination Go object. Destination can be a struct, slice or channel. // // Mapping to quads is done via Go struct tag "quad" or "json" as a fallback. // // A simplest mapping is an "@id" tag which saves node ID (subject of a quad) into tagged field. // // type Node struct{ // ID quad.IRI `json:"@id"` // or `quad:"@id"` // } // // Field with an "@id" tag is omitted, but in case of Go->quads mapping new ID will be generated // using GenerateID callback, which can be changed to provide a custom mappings. // // All other tags are interpreted as a predicate name for a specific field: // // type Person struct{ // ID quad.IRI `json:"@id"` // Name string `json:"name"` // } // p := Person{"bob","Bob"} // // is equivalent to triple: // // "Bob" // // Predicate IRIs in RDF can have a long namespaces, but they can be written in short // form. They will be expanded automatically if namespace prefix is registered within // QuadStore or globally via "voc" package. // There is also a special predicate name "@type" which is mapped to "rdf:type" IRI. // // voc.RegisterPrefix("ex:", "http://example.org/") // type Person struct{ // ID quad.IRI `json:"@id"` // Type quad.IRI `json:"@type"` // Name string `json:"ex:name"` // will be expanded to http://example.org/name // } // p := Person{"bob",quad.IRI("Person"),"Bob"} // // is equivalent to triples: // // // // "Bob" // // Predicate link direction can be reversed with a special tag syntax (not available for "json" tag): // // type Person struct{ // ID quad.IRI `json:"@id"` // Name string `json:"name"` // same as `quad:"name"` or `quad:"name > *"` // Parents []quad.IRI `quad:"isParentOf < *"` // } // p := Person{"bob","Bob",[]quad.IRI{"alice","fred"}} // // is equivalent to triples: // // "Bob" // // // // // // All fields in structs are interpreted as required (except slices), thus struct will not be // loaded if one of fields is missing. An "optional" tag can be specified to relax this requirement. // Also, "required" can be specified for slices to alter default value. // // type Person struct{ // ID quad.IRI `json:"@id"` // Name string `json:"name"` // required field // ThirdName string `quad:"thirdName,optional"` // can be empty // FollowedBy []quad.IRI `quad:"follows"` // } func (c *Config) LoadTo(ctx context.Context, qs graph.QuadStore, dst interface{}, ids ...quad.Value) error { return c.LoadToDepth(ctx, qs, dst, -1, ids...) } // LoadToDepth is the same as LoadTo, but stops at a specified depth. // Negative value means unlimited depth, and zero means top level only. func (c *Config) LoadToDepth(ctx context.Context, qs graph.QuadStore, dst interface{}, depth int, ids ...quad.Value) error { if dst == nil { return fmt.Errorf("nil destination object") } var it iterator.Shape if len(ids) != 0 { fixed := iterator.NewFixed() for _, id := range ids { idv, err := qs.ValueOf(id) if err != nil { return err } fixed.Add(idv) } it = fixed } var rv reflect.Value if v, ok := dst.(reflect.Value); ok { rv = v } else { rv = reflect.ValueOf(dst) } return c.LoadIteratorToDepth(ctx, qs, rv, depth, it) } // LoadPathTo is the same as LoadTo, but starts loading objects from a given path. func (c *Config) LoadPathTo(ctx context.Context, qs graph.QuadStore, dst interface{}, p *path.Path) error { return c.LoadIteratorTo(ctx, qs, reflect.ValueOf(dst), p.BuildIterator(ctx)) } // LoadIteratorTo is a lower level version of LoadTo. // // It expects an iterator of nodes to be passed explicitly and // destination value to be obtained via reflect package manually. // // Nodes iterator can be nil, All iterator will be used in this case. func (c *Config) LoadIteratorTo(ctx context.Context, qs graph.QuadStore, dst reflect.Value, list iterator.Shape) error { return c.LoadIteratorToDepth(ctx, qs, dst, -1, list) } // LoadIteratorToDepth is the same as LoadIteratorTo, but stops at a specified depth. // Negative value means unlimited depth, and zero means top level only. func (c *Config) LoadIteratorToDepth(ctx context.Context, qs graph.QuadStore, dst reflect.Value, depth int, list iterator.Shape) error { if depth >= 0 { // 0 depth means "current level only" for user, but it's easier to make depth=0 a stop condition depth++ } l := c.newLoader(qs) return l.loadIteratorToDepth(ctx, dst, depth, list) } type loader struct { c *Config qs graph.QuadStore pathForType map[reflect.Type]*path.Path pathForTypeRoot map[reflect.Type]*path.Path seen map[quad.Value]reflect.Value } func (c *Config) newLoader(qs graph.QuadStore) *loader { return &loader{ c: c, qs: qs, pathForType: make(map[reflect.Type]*path.Path), pathForTypeRoot: make(map[reflect.Type]*path.Path), seen: make(map[quad.Value]reflect.Value), } } func (l *loader) makePathForType(rt reflect.Type, tagPref string, rootOnly bool) (*path.Path, error) { for rt.Kind() == reflect.Ptr { rt = rt.Elem() } if rt.Kind() != reflect.Struct { return nil, fmt.Errorf("expected struct, got %v", rt) } if tagPref == "" { m := l.pathForType if rootOnly { m = l.pathForTypeRoot } if p, ok := m[rt]; ok { return p, nil } } p := path.StartMorphism() if iri := getTypeIRI(rt); iri != quad.IRI("") { p = p.Has(l.c.iri(iriType), iri) } // TODO(dennwc): rewrite to shapes allOptional := true var alt *path.Path for i := 0; i < rt.NumField(); i++ { f := rt.Field(i) if f.Anonymous { pa, err := l.makePathForType(f.Type, tagPref+f.Name+".", rootOnly) if err != nil { return nil, err } p = p.Follow(pa) continue } name := f.Name rule, err := l.c.fieldRule(f) if err != nil { return nil, err } else if rule == nil { // skip continue } ft := f.Type if ft.Kind() == reflect.Ptr { ft = ft.Elem() } if err = checkFieldType(ft); err != nil { return nil, err } switch rule := rule.(type) { case idRule: p = p.Tag(tagPref + name) case constraintRule: allOptional = false var nodes []quad.Value if rule.Val != "" { nodes = []quad.Value{rule.Val} } if rule.Rev { p = p.HasReverse(rule.Pred, nodes...) } else { p = p.Has(rule.Pred, nodes...) } case saveRule: tag := tagPref + name if rule.Opt { if !rootOnly { if rule.Rev { p = p.SaveOptionalReverse(rule.Pred, tag) if allOptional { ap := path.StartMorphism().HasReverse(rule.Pred) if alt == nil { alt = ap } else { alt = alt.Or(ap) } } } else { p = p.SaveOptional(rule.Pred, tag) if allOptional { ap := path.StartMorphism().Has(rule.Pred) if alt == nil { alt = ap } else { alt = alt.Or(ap) } } } } } else if rootOnly { // do not save field, enforce constraint only allOptional = false if rule.Rev { p = p.HasReverse(rule.Pred) } else { p = p.Has(rule.Pred) } } else { allOptional = false if rule.Rev { p = p.SaveReverse(rule.Pred, tag) } else { p = p.Save(rule.Pred, tag) } } } } if allOptional { p = p.And(alt.Unique()) } if tagPref != "" { return p, nil } m := l.pathForType if rootOnly { m = l.pathForTypeRoot } m[rt] = p return p, nil } func (l *loader) loadToValue(ctx context.Context, dst reflect.Value, depth int, m map[string][]graph.Ref, tagPref string) error { if ctx == nil { ctx = context.TODO() } for dst.Kind() == reflect.Ptr { dst = dst.Elem() } rt := dst.Type() if rt.Kind() != reflect.Struct { return fmt.Errorf("expected struct, got %v", rt) } var fields fieldRules if v := ctx.Value(fieldsCtxKey{}); v != nil { fields = v.(fieldRules) } else { nfields, err := l.c.rulesFor(rt) if err != nil { return err } fields = nfields } if depth != 0 { // do not check required fields if depth limit is reached for name, field := range fields { if r, ok := field.(saveRule); ok && !r.Opt { if vals := m[name]; len(vals) == 0 { return errRequiredFieldIsMissing } } } } for i := 0; i < rt.NumField(); i++ { select { case <-ctx.Done(): return ctx.Err() default: } f := rt.Field(i) name := f.Name if err := checkFieldType(f.Type); err != nil { return err } df := dst.Field(i) if f.Anonymous { if err := l.loadToValue(ctx, df, depth, m, tagPref+name+"."); err != nil { return fmt.Errorf("load anonymous field %s failed: %v", f.Name, err) } continue } rules := fields[tagPref+name] if rules == nil { continue } arr, ok := m[tagPref+name] if !ok || len(arr) == 0 { continue } ft := f.Type native := isNative(ft) ptr := ft.Kind() == reflect.Ptr for ft.Kind() == reflect.Ptr || ft.Kind() == reflect.Slice { ft = ft.Elem() native = native || isNative(ft) switch ft.Kind() { case reflect.Ptr: ptr = true case reflect.Slice: ptr = false } } recursive := !native && ft.Kind() == reflect.Struct for _, fv := range arr { var sv reflect.Value if recursive { if ptr { fv, err := l.qs.NameOf(fv) if err != nil { return err } var ok bool sv, ok = l.seen[fv] if ok && sv.Type().AssignableTo(f.Type) { df.Set(sv) continue } } sv = reflect.New(ft).Elem() err := l.loadIteratorToDepth(ctx, sv, depth-1, iterator.NewFixed(fv)) if err == errRequiredFieldIsMissing { continue } else if err != nil { return err } } else { fv, err := l.qs.NameOf(fv) if err != nil { return err } if fv == nil { continue } sv = reflect.ValueOf(fv) } if err := DefaultConverter.SetValue(df, sv); err != nil { return fmt.Errorf("field %s: %v", f.Name, err) } } } return nil } func (l *loader) iteratorForType(ctx context.Context, root iterator.Shape, rt reflect.Type, rootOnly bool) (iterator.Shape, error) { p, err := l.makePathForType(rt, "", rootOnly) if err != nil { return nil, err } return l.iteratorFromPath(ctx, root, p) } func mergeMap(dst map[string][]graph.Ref, m map[string]graph.Ref) { loop: for k, v := range m { sl := dst[k] for _, sv := range sl { if keysEqual(sv, v) { continue loop } } dst[k] = append(sl, v) } } func (l *loader) loadIteratorToDepth(ctx context.Context, dst reflect.Value, depth int, list iterator.Shape) error { if ctx == nil { ctx = context.TODO() } if dst.Kind() == reflect.Ptr { dst = dst.Elem() } et := dst.Type() slice, chanl := false, false if dst.Kind() == reflect.Slice { et = et.Elem() slice = true } else if dst.Kind() == reflect.Chan { et = et.Elem() chanl = true defer dst.Close() } fields, err := l.c.rulesFor(et) if err != nil { return err } ctxDone := func() bool { select { case <-ctx.Done(): return true default: } return false } if ctxDone() { return ctx.Err() } rootOnly := depth == 0 its, err := l.iteratorForType(ctx, list, et, rootOnly) if err != nil { return err } it := its.Iterate() defer it.Close() ctx = context.WithValue(ctx, fieldsCtxKey{}, fields) for it.Next(ctx) { if ctxDone() { return ctx.Err() } id, err := l.qs.NameOf(it.Result()) if err != nil { return err } if id != nil { if sv, ok := l.seen[id]; ok { if slice { dst.Set(reflect.Append(dst, sv.Elem())) } else if chanl { dst.Send(sv.Elem()) } else if dst.Kind() != reflect.Ptr { dst.Set(sv.Elem()) return nil } else { dst.Set(sv) return nil } continue } } mp := make(map[string]graph.Ref) it.TagResults(mp) if len(mp) == 0 { continue } cur := dst if slice || chanl { cur = reflect.New(et) } mo := make(map[string][]graph.Ref, len(mp)) for k, v := range mp { mo[k] = []graph.Ref{v} } for it.NextPath(ctx) { if ctxDone() { return ctx.Err() } mp = make(map[string]graph.Ref) it.TagResults(mp) if len(mp) == 0 { continue } // TODO(dennwc): replace with something more efficient mergeMap(mo, mp) } if id != nil { sv := cur if sv.Kind() != reflect.Ptr && sv.CanAddr() { sv = sv.Addr() } l.seen[id] = sv } err = l.loadToValue(ctx, cur, depth, mo, "") if err == errRequiredFieldIsMissing { if !slice && !chanl { return err } continue } else if err != nil { return err } if slice { dst.Set(reflect.Append(dst, cur.Elem())) } else if chanl { dst.Send(cur.Elem()) } else { return nil } } if err := it.Err(); err != nil { return err } if slice || chanl { return nil } if list != nil { // TODO(dennwc): optional optimization: do this only if iterator is not "all nodes" // distinguish between missing object and type constraints and := iterator.NewAnd(list, l.qs.NodesAllIterator()).Iterate() defer and.Close() if and.Next(ctx) { return errRequiredFieldIsMissing } } return errNotFound } func (l *loader) iteratorFromPath(ctx context.Context, root iterator.Shape, p *path.Path) (iterator.Shape, error) { it := p.BuildIteratorOn(ctx, l.qs) if root != nil { it = iterator.NewAnd(root, it) } if Optimize { it, _ = it.Optimize(ctx) } return it, nil } ================================================ FILE: schema/loader_test.go ================================================ package schema_test import ( "reflect" "testing" "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" "github.com/stretchr/testify/require" ) func TestLoadLoop(t *testing.T) { sch := schema.NewConfig() a := &NodeLoop{ID: iri("A"), Name: "Node A"} a.Next = a qs := memstore.New([]quad.Quad{ {a.ID, iri("name"), quad.String(a.Name), nil}, {a.ID, iri("next"), a.ID, nil}, }...) b := &NodeLoop{} if err := sch.LoadIteratorTo(nil, qs, reflect.ValueOf(b), nil); err != nil { t.Error(err) return } if a.ID != b.ID || a.Name != b.Name { t.Fatalf("%#v vs %#v", a, b) } if b != b.Next { t.Fatalf("loop is broken: %p vs %p", b, b.Next) } a = &NodeLoop{ID: iri("A"), Name: "Node A"} b = &NodeLoop{ID: iri("B"), Name: "Node B"} c := &NodeLoop{ID: iri("C"), Name: "Node C"} a.Next = b b.Next = c c.Next = a qs = memstore.New([]quad.Quad{ {a.ID, iri("name"), quad.String(a.Name), nil}, {b.ID, iri("name"), quad.String(b.Name), nil}, {c.ID, iri("name"), quad.String(c.Name), nil}, {a.ID, iri("next"), b.ID, nil}, {b.ID, iri("next"), c.ID, nil}, {c.ID, iri("next"), a.ID, nil}, }...) a1 := &NodeLoop{} if err := sch.LoadIteratorTo(nil, qs, reflect.ValueOf(a1), nil); err != nil { t.Error(err) return } if a.ID != a1.ID || a.Name != a1.Name { t.Fatalf("%#v vs %#v", a, b) } b1 := a1.Next c1 := b1.Next if b.ID != b1.ID || b.Name != b1.Name { t.Fatalf("%#v vs %#v", a, b) } if c.ID != c1.ID || c.Name != c1.Name { t.Fatalf("%#v vs %#v", a, b) } if a1 != c1.Next { t.Fatalf("loop is broken: %p vs %p", a1, c1.Next) } } func TestLoadIteratorTo(t *testing.T) { sch := schema.NewConfig() for i, c := range testFillValueCases { t.Run(c.name, func(t *testing.T) { qs := memstore.New(c.quads...) rt := reflect.TypeOf(c.expect) var out reflect.Value if rt.Kind() == reflect.Ptr { out = reflect.New(rt.Elem()) } else { out = reflect.New(rt) } var it iterator.Shape if c.from != nil { fixed := iterator.NewFixed() for _, id := range c.from { qsv, err := qs.ValueOf(id) require.NoError(t, err) fixed.Add(qsv) } it = fixed } depth := c.depth if depth == 0 { depth = -1 } if err := sch.LoadIteratorToDepth(nil, qs, out, depth, it); err != nil { t.Errorf("case %d failed: %v", i+1, err) return } var got interface{} if rt.Kind() == reflect.Ptr { got = out.Interface() } else { got = out.Elem().Interface() } if s, ok := got.(interface { Sort() }); ok { s.Sort() } if s, ok := c.expect.(interface { Sort() }); ok { s.Sort() } if !reflect.DeepEqual(got, c.expect) { t.Errorf("case %d failed: objects are different\n%#v\n%#v", i+1, got, c.expect, ) } }) } } var testFillValueCases = []struct { name string expect interface{} quads []quad.Quad depth int from []quad.Value }{ { name: "complex object", expect: struct { rdfType struct{} `quad:"rdf:type > some:Type"` ID quad.IRI `quad:"@id"` Name string `quad:"name"` Values []string `quad:"values"` Items []item `quad:"items"` Sub *item `quad:"sub"` Val int `quad:"val"` }{ ID: "1234", Name: "some item", Values: []string{"val1", "val2"}, Items: []item{ {ID: "sub1", Name: "Sub 1"}, {ID: "sub2", Name: "Sub 2"}, }, Sub: &item{ID: "sub3", Name: "Sub 3"}, Val: 123, }, quads: []quad.Quad{ {iri("1234"), typeIRI, iri("some:Type"), nil}, {iri("1234"), iri("name"), quad.String("some item"), nil}, {iri("1234"), iri("values"), quad.String("val1"), nil}, {iri("1234"), iri("values"), quad.String("val2"), nil}, {iri("sub1"), typeIRI, iri("some:item"), nil}, {iri("sub1"), iri("name"), quad.String("Sub 1"), nil}, {iri("1234"), iri("items"), iri("sub1"), nil}, {iri("sub2"), typeIRI, iri("some:item"), nil}, {iri("sub2"), iri("name"), quad.String("Sub 2"), nil}, {iri("1234"), iri("items"), iri("sub2"), nil}, {iri("sub3"), typeIRI, iri("some:item"), nil}, {iri("sub3"), iri("name"), quad.String("Sub 3"), nil}, {iri("1234"), iri("sub"), iri("sub3"), nil}, {iri("1234"), iri("val"), quad.Int(123), nil}, }, }, { name: "complex object (id value)", expect: struct { rdfType struct{} `quad:"rdf:type > some:Type"` ID quad.Value `quad:"@id"` Name string `quad:"name"` Values []string `quad:"values"` Items []item `quad:"items"` }{ ID: quad.BNode("1234"), Name: "some item", Values: []string{"val1", "val2"}, Items: []item{ {ID: "sub1", Name: "Sub 1"}, {ID: "sub2", Name: "Sub 2"}, }, }, quads: []quad.Quad{ {quad.BNode("1234"), typeIRI, iri("some:Type"), nil}, {quad.BNode("1234"), iri("name"), quad.String("some item"), nil}, {quad.BNode("1234"), iri("values"), quad.String("val1"), nil}, {quad.BNode("1234"), iri("values"), quad.String("val2"), nil}, {iri("sub1"), typeIRI, iri("some:item"), nil}, {iri("sub1"), iri("name"), quad.String("Sub 1"), nil}, {quad.BNode("1234"), iri("items"), iri("sub1"), nil}, {iri("sub2"), typeIRI, iri("some:item"), nil}, {iri("sub2"), iri("name"), quad.String("Sub 2"), nil}, {quad.BNode("1234"), iri("items"), iri("sub2"), nil}, }, }, { name: "embedded object", expect: struct { rdfType struct{} `quad:"rdf:type > some:Type"` item2 ID quad.IRI `quad:"@id"` Values []string `quad:"values"` }{ item2: item2{Name: "Sub 1", Spec: "special"}, ID: "1234", Values: []string{"val1", "val2"}, }, quads: []quad.Quad{ {iri("1234"), typeIRI, iri("some:Type"), nil}, {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, {iri("1234"), iri("spec"), quad.String("special"), nil}, {iri("1234"), iri("values"), quad.String("val1"), nil}, {iri("1234"), iri("values"), quad.String("val2"), nil}, }, }, { name: "type shorthand", expect: struct { rdfType struct{} `quad:"@type > some:Type"` item2 ID quad.IRI `quad:"@id"` Values []string `quad:"values"` }{ item2: item2{Name: "Sub 1", Spec: "special"}, ID: "1234", Values: []string{"val1", "val2"}, }, quads: []quad.Quad{ {iri("1234"), typeIRI, iri("some:Type"), nil}, {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, {iri("1234"), iri("spec"), quad.String("special"), nil}, {iri("1234"), iri("values"), quad.String("val1"), nil}, {iri("1234"), iri("values"), quad.String("val2"), nil}, }, }, { name: "tree", expect: treeItem{ ID: iri("n1"), Name: "Node 1", Children: []treeItem{ { ID: iri("n2"), Name: "Node 2", }, { ID: iri("n3"), Name: "Node 3", Children: []treeItem{ { ID: iri("n4"), Name: "Node 4", }, }, }, }, }, quads: treeQuads, from: []quad.Value{iri("n1")}, }, { name: "tree with depth limit 1", expect: treeItem{ ID: iri("n1"), Name: "Node 1", Children: []treeItem{ { ID: iri("n2"), Name: "Node 2", }, { ID: iri("n3"), Name: "Node 3", Children: []treeItem{ { ID: iri("n4"), }, }, }, }, }, depth: 1, quads: treeQuads, from: []quad.Value{iri("n1")}, }, { name: "tree with depth limit 2", expect: treeItemOpt{ ID: iri("n1"), Name: "Node 1", Children: []treeItemOpt{ { ID: iri("n2"), Name: "Node 2", }, { ID: iri("n3"), Name: "Node 3", Children: []treeItemOpt{ { ID: iri("n4"), Name: "Node 4", }, }, }, }, }, depth: 2, quads: treeQuads, from: []quad.Value{iri("n1")}, }, { name: "tree with required children", expect: treeItemReq{ ID: iri("n1"), Name: "Node 1", Children: []treeItemReq{ { ID: iri("n3"), Name: "Node 3", // TODO(dennwc): a strange behavior: this field is required, but it's empty for current object, // because all it's children are missing the same field. Leaving this as-is for now because // it's weird to set Children field as required in a tree. Children: nil, }, }, }, quads: treeQuads, from: []quad.Value{iri("n1")}, }, { name: "simple object", expect: subObject{ genObject: genObject{ ID: "1234", Name: "Obj", }, Num: 3, }, quads: []quad.Quad{ {iri("1234"), iri("name"), quad.String("Obj"), nil}, {iri("1234"), iri("num"), quad.Int(3), nil}, }, }, { name: "typedef", expect: genObjectTypedef{ ID: "1234", Name: "Obj", }, quads: []quad.Quad{ {iri("1234"), iri("name"), quad.String("Obj"), nil}, }, }, { name: "coords", expect: Coords{Lat: 12.3, Lng: 34.5}, quads: []quad.Quad{ {iri("c1"), typeIRI, iri("ex:Coords"), nil}, {iri("c1"), iri("ex:lat"), quad.Float(12.3), nil}, {iri("c1"), iri("ex:lng"), quad.Float(34.5), nil}, }, }, { name: "same node", expect: NestedNode{ ID: "c1", Name: "A", Prev: genObject{ ID: "c2", Name: "B", }, Next: genObject{ ID: "c2", Name: "B", }, }, quads: []quad.Quad{ {iri("c1"), iri("name"), quad.String("A"), nil}, {iri("c2"), iri("name"), quad.String("B"), nil}, {iri("c1"), iri("next"), iri("c2"), nil}, {iri("c1"), iri("prev"), iri("c2"), nil}, }, }, { name: "all optional", expect: Alts{ Alt: []OptFields{ {One: "A"}, {Two: "B"}, {One: "C", Two: "D"}, }, }, quads: []quad.Quad{ {iri("c1"), iri("alt"), iri("h1"), nil}, {iri("c1"), iri("alt"), iri("h2"), nil}, {iri("c1"), iri("alt"), iri("h3"), nil}, {iri("h1"), iri("one"), quad.String("A"), nil}, {iri("h2"), iri("two"), quad.String("B"), nil}, {iri("h3"), iri("one"), quad.String("C"), nil}, {iri("h3"), iri("two"), quad.String("D"), nil}, }, }, } ================================================ FILE: schema/namespaces.go ================================================ package schema import ( "context" "fmt" "reflect" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) type namespace struct { _ struct{} `quad:"@type > cayley:namespace"` Full quad.IRI `quad:"@id"` Prefix quad.IRI `quad:"cayley:prefix"` } // WriteNamespaces will writes namespaces list into graph. func (c *Config) WriteNamespaces(w quad.Writer, n *voc.Namespaces) error { rules, err := c.rulesFor(reflect.TypeOf(namespace{})) if err != nil { return fmt.Errorf("can't load rules: %v", err) } wr := c.newWriter(w) for _, ns := range n.List() { obj := namespace{ Full: quad.IRI(ns.Full), Prefix: quad.IRI(ns.Prefix), } rv := reflect.ValueOf(obj) if err = wr.writeValueAs(obj.Full, rv, "", rules); err != nil { return err } } return nil } // LoadNamespaces will load namespaces stored in graph to a specified list. // If destination list is empty, global namespace registry will be used. func (c *Config) LoadNamespaces(ctx context.Context, qs graph.QuadStore, dest *voc.Namespaces) error { var list []namespace if err := c.LoadTo(ctx, qs, &list); err != nil { return err } register := dest.Register if dest == nil { register = voc.Register } for _, ns := range list { register(voc.Namespace{ Prefix: string(ns.Prefix), Full: string(ns.Full), }) } return nil } ================================================ FILE: schema/namespaces_test.go ================================================ package schema_test import ( "context" "reflect" "sort" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) func TestSaveNamespaces(t *testing.T) { sch := schema.NewConfig() save := []voc.Namespace{ {Full: "http://example.org/", Prefix: "ex:"}, {Full: "http://cayley.io/", Prefix: "c:"}, } var ns voc.Namespaces for _, n := range save { ns.Register(n) } qs := memstore.New() err := sch.WriteNamespaces(qs, &ns) if err != nil { t.Fatal(err) } var ns2 voc.Namespaces err = sch.LoadNamespaces(context.TODO(), qs, &ns2) if err != nil { t.Fatal(err) } got := ns2.List() sort.Sort(voc.ByFullName(save)) sort.Sort(voc.ByFullName(got)) if !reflect.DeepEqual(save, got) { t.Fatalf("wrong namespaces returned: got: %v, expect: %v", got, save) } qr := graph.NewQuadStoreReader(qs) q, err := quad.ReadAll(qr) qr.Close() if err != nil { t.Fatal(err) } expect := []quad.Quad{ quad.MakeIRI("http://cayley.io/", "cayley:prefix", "c:", ""), quad.MakeIRI("http://cayley.io/", "rdf:type", "cayley:namespace", ""), quad.MakeIRI("http://example.org/", "cayley:prefix", "ex:", ""), quad.MakeIRI("http://example.org/", "rdf:type", "cayley:namespace", ""), } sort.Sort(quad.ByQuadString(expect)) sort.Sort(quad.ByQuadString(q)) if !reflect.DeepEqual(expect, q) { t.Fatalf("wrong quads returned: got: %v, expect: %v", q, expect) } } ================================================ FILE: schema/schema.go ================================================ // Package schema contains helpers to map Go objects to quads and vise-versa. // // This package is not a full schema library. It will not save or force any // RDF schema constrains, it only provides a mapping. package schema import ( "fmt" "reflect" "strings" "sync" "unicode" "unicode/utf8" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc/rdf" ) var global = NewConfig() // Global returns a default global schema config. func Global() *Config { return global } var reflQuadValue = reflect.TypeOf((*quad.Value)(nil)).Elem() type ErrReqFieldNotSet struct { Field string } func (e ErrReqFieldNotSet) Error() string { return fmt.Sprintf("required field is not set: %s", e.Field) } // IRIMode controls how IRIs are processed. type IRIMode int const ( // IRINative applies no transformation to IRIs. IRINative = IRIMode(iota) // IRIShort will compact all IRIs with known namespaces. IRIShort // IRIFull will expand all IRIs with known namespaces. IRIFull ) // NewConfig creates a new schema config. func NewConfig() *Config { return &Config{ IRIs: IRINative, } } // Config controls behavior of schema package. type Config struct { // IRIs set a conversion mode for all IRIs. IRIs IRIMode // GenerateID is called when any object without an ID field is being saved. GenerateID func(_ interface{}) quad.Value // Label will be added to all quads written. Does not affect queries. Label quad.Value rulesForTypeMu sync.RWMutex rulesForType map[reflect.Type]fieldRules } func (c *Config) genID(o interface{}) quad.Value { gen := c.GenerateID if gen == nil { gen = func(_ interface{}) quad.Value { return quad.RandomBlankNode() } } return gen(o) } type rule interface { isRule() } type constraintRule struct { Pred quad.IRI Val quad.IRI Rev bool } func (constraintRule) isRule() {} type saveRule struct { Pred quad.IRI Rev bool Opt bool } func (saveRule) isRule() {} type idRule struct{} func (idRule) isRule() {} const iriType = quad.IRI(rdf.Type) func (c *Config) iri(v quad.IRI) quad.IRI { switch c.IRIs { case IRIShort: v = v.Short() case IRIFull: v = v.Full() } return v } func (c *Config) toIRI(s string) quad.IRI { var v quad.IRI if s == "@type" { v = iriType } else { v = quad.IRI(s) } return c.iri(v) } var reflEmptyStruct = reflect.TypeOf(struct{}{}) func (c *Config) fieldRule(fld reflect.StructField) (rule, error) { tag := fld.Tag.Get("quad") sub := strings.Split(tag, ",") tag, sub = sub[0], sub[1:] const ( trim = ` ` spo, ops = `>`, `<` any, none = `*`, `-` this = `@id` ) tag = strings.Trim(tag, trim) jsn := false if tag == "" { tag = strings.SplitN(fld.Tag.Get("json"), ",", 2)[0] jsn = true } if tag == "" || tag == none { return nil, nil // ignore } rule := strings.Trim(tag, trim) if rule == this { return idRule{}, nil } opt := false req := false for _, s := range sub { if s == "opt" || s == "optional" { opt = true } if s == "req" || s == "required" { req = true } } if req { opt = false } else if fld.Type.Kind() == reflect.Slice { opt = true } rev := strings.Contains(rule, ops) var tri []string if jsn { tri = []string{rule} } else if rev { // oo // default tri = strings.SplitN(rule, spo, 3) if len(tri) > 2 { //len(tri) != 2 { return nil, fmt.Errorf("wrong quad tag format: '%s'", rule) } } var ps, vs string if rev { ps, vs = strings.Trim(tri[0], trim), strings.Trim(tri[1], trim) } else { ps, vs = strings.Trim(tri[0], trim), any if len(tri) > 1 { vs = strings.Trim(tri[1], trim) } } if ps == "" { return nil, fmt.Errorf("wrong quad format: '%s': no predicate", rule) } p := c.toIRI(ps) if vs == "" || vs == any && fld.Type != reflEmptyStruct { return saveRule{Pred: p, Rev: rev, Opt: opt}, nil } return constraintRule{Pred: p, Val: c.toIRI(vs), Rev: rev}, nil } func checkFieldType(ftp reflect.Type) error { for ftp.Kind() == reflect.Ptr || ftp.Kind() == reflect.Slice { ftp = ftp.Elem() } switch ftp.Kind() { case reflect.Array: // TODO: support arrays return fmt.Errorf("array fields are not supported yet") case reflect.Func, reflect.Invalid: return fmt.Errorf("%v fields are not supported", ftp.Kind()) default: } return nil } var ( typesMu sync.RWMutex typeToIRI = make(map[reflect.Type]quad.IRI) iriToType = make(map[quad.IRI]reflect.Type) ) func getTypeIRI(rt reflect.Type) quad.IRI { typesMu.RLock() iri := typeToIRI[rt] typesMu.RUnlock() return iri } // RegisterType associates an IRI with a given Go type. // // All queries and writes will require or add a type triple. func RegisterType(iri quad.IRI, obj interface{}) { var rt reflect.Type if obj != nil { if t, ok := obj.(reflect.Type); ok { rt = t } else { rt = reflect.TypeOf(obj) if rt.Kind() == reflect.Ptr { rt = rt.Elem() } } } full := iri.Full() typesMu.Lock() defer typesMu.Unlock() if obj == nil { tp := iriToType[full] delete(typeToIRI, tp) delete(iriToType, full) return } if _, exists := typeToIRI[rt]; exists { panic(fmt.Errorf("type %v is already registered", rt)) } if _, exists := iriToType[full]; exists { panic(fmt.Errorf("IRI %v is already registered", iri)) } typeToIRI[rt] = iri iriToType[full] = rt } // PathForType builds a path (morphism) for a given Go type. func (c *Config) PathForType(rt reflect.Type) (*path.Path, error) { l := c.newLoader(nil) return l.makePathForType(rt, "", false) } func anonFieldType(fld reflect.StructField) (reflect.Type, bool) { ft := fld.Type if ft.Kind() == reflect.Ptr { ft = ft.Elem() } if ft.Kind() == reflect.Struct { return ft, true } return ft, false } func (c *Config) rulesForStructTo(out fieldRules, pref string, rt reflect.Type) error { for i := 0; i < rt.NumField(); i++ { f := rt.Field(i) name := f.Name if f.Anonymous { if ft, ok := anonFieldType(f); !ok { return fmt.Errorf("anonymous fields of type %v are not supported", ft) } else if err := c.rulesForStructTo(out, pref+name+".", ft); err != nil { return err } continue } rules, err := c.fieldRule(f) if err != nil { return err } if rules != nil { out[pref+name] = rules } } return nil } // rulesFor // // Returned map should not be changed. func (c *Config) rulesFor(rt reflect.Type) (fieldRules, error) { // if rt.Kind() != reflect.Struct { // return nil, fmt.Errorf("expected struct, got: %v", rt) // } c.rulesForTypeMu.RLock() rules, ok := c.rulesForType[rt] c.rulesForTypeMu.RUnlock() if ok { return rules, nil } out := make(fieldRules) if err := c.rulesForStructTo(out, "", rt); err != nil { return nil, err } c.rulesForTypeMu.Lock() if c.rulesForType == nil { c.rulesForType = make(map[reflect.Type]fieldRules) } c.rulesForType[rt] = out c.rulesForTypeMu.Unlock() return out, nil } type fieldsCtxKey struct{} type fieldRules map[string]rule type ValueConverter interface { SetValue(dst reflect.Value, src reflect.Value) error } type ValueConverterFunc func(dst reflect.Value, src reflect.Value) error func (f ValueConverterFunc) SetValue(dst reflect.Value, src reflect.Value) error { return f(dst, src) } var DefaultConverter ValueConverter type ErrTypeConversionFailed struct { From reflect.Type To reflect.Type } func (e ErrTypeConversionFailed) Error() string { return fmt.Sprintf("cannot convert %v to %v", e.From, e.To) } func init() { DefaultConverter = ValueConverterFunc(func(dst reflect.Value, src reflect.Value) error { dt, st := dst.Type(), src.Type() if dt == st || (dt.Kind() == reflect.Interface && st.Implements(dt)) { dst.Set(src) return nil } else if st.ConvertibleTo(dt) { dst.Set(src.Convert(dt)) return nil } else if dt.Kind() == reflect.Ptr { v := reflect.New(dt.Elem()) if err := DefaultConverter.SetValue(v.Elem(), src); err != nil { return err } dst.Set(v) return nil } else if dt.Kind() == reflect.Slice { v := reflect.New(dt.Elem()) if err := DefaultConverter.SetValue(v.Elem(), src); err != nil { return err } dst.Set(reflect.Append(dst, v.Elem())) return nil } return ErrTypeConversionFailed{From: src.Type(), To: dst.Type()} }) } func isNative(rt reflect.Type) bool { // TODO(dennwc): replace _, ok := quad.AsValue(reflect.Zero(rt).Interface()) return ok } func keysEqual(v1, v2 graph.Ref) bool { type key interface { Key() interface{} } e1, ok1 := v1.(key) e2, ok2 := v2.(key) if ok1 != ok2 { return false } if ok1 && ok2 { return e1.Key() == e2.Key() } return v1 == v2 } func isExported(name string) bool { ch, _ := utf8.DecodeRuneInString(name) return unicode.IsUpper(ch) } func isZero(rv reflect.Value) bool { switch rv.Kind() { case reflect.Ptr: return rv.IsNil() case reflect.Slice, reflect.Map: return rv.IsNil() || rv.Len() == 0 case reflect.Struct: // have to be careful here - struct may contain slice fields, // so we cannot compare them directly rt := rv.Type() exported := 0 for i := 0; i < rt.NumField(); i++ { f := rt.Field(i) if !isExported(f.Name) { continue } exported++ if !isZero(rv.Field(i)) { return false } } if exported != 0 { return true } // opaque type - compare directly } // primitive types return rv.Interface() == reflect.Zero(rv.Type()).Interface() } func (c *Config) idFor(rules fieldRules, rt reflect.Type, rv reflect.Value, pref string) (id quad.Value, err error) { hasAnon := false for i := 0; i < rt.NumField(); i++ { fld := rt.Field(i) hasAnon = hasAnon || fld.Anonymous if _, ok := rules[pref+fld.Name].(idRule); ok { vid := rv.Field(i).Interface() switch vid := vid.(type) { case quad.IRI: id = c.iri(vid) case quad.BNode: id = vid case string: id = c.toIRI(vid) default: err = fmt.Errorf("unsupported type for id field: %T", vid) } return } } if !hasAnon { return } // second pass - look for anonymous fields for i := 0; i < rt.NumField(); i++ { fld := rt.Field(i) if !fld.Anonymous { continue } id, err = c.idFor(rules, fld.Type, rv.Field(i), pref+fld.Name+".") if err != nil || id != nil { return } } return } ================================================ FILE: schema/schema_test.go ================================================ package schema_test import ( "sort" "time" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" "github.com/cayleygraph/quad/voc/rdf" ) func init() { voc.RegisterPrefix("ex:", "http://example.org/") schema.RegisterType(quad.IRI("ex:Coords"), Coords{}) } type item struct { rdfType struct{} `quad:"rdf:type > some:item"` ID quad.IRI `quad:"@id"` Name string `quad:"name"` Spec string `quad:"spec,optional"` } type item2 struct { Name string `quad:"name"` Spec string `quad:"spec"` } type timeItem struct { ID quad.IRI `quad:"@id"` Name string `quad:"name"` TS time.Time `quad:"ts"` } type treeItemByIRI []treeItem func (o treeItemByIRI) Len() int { return len(o) } func (o treeItemByIRI) Less(i, j int) bool { return o[i].ID < o[j].ID } func (o treeItemByIRI) Swap(i, j int) { o[i], o[j] = o[j], o[i] } type treeItem struct { ID quad.IRI `quad:"@id"` Name string `quad:"name"` Children []treeItem `quad:"child"` } func (t *treeItem) Sort() { for _, c := range t.Children { c.Sort() } sort.Sort(treeItemByIRI(t.Children)) } type treeItemOptByIRI []treeItemOpt func (o treeItemOptByIRI) Len() int { return len(o) } func (o treeItemOptByIRI) Less(i, j int) bool { return o[i].ID < o[j].ID } func (o treeItemOptByIRI) Swap(i, j int) { o[i], o[j] = o[j], o[i] } type treeItemOpt struct { ID quad.IRI `quad:"@id"` Name string `quad:"name"` Children []treeItemOpt `quad:"child,optional"` } func (t *treeItemOpt) Sort() { for _, c := range t.Children { c.Sort() } sort.Sort(treeItemOptByIRI(t.Children)) } type treeItemReqByIRI []treeItemReq func (o treeItemReqByIRI) Len() int { return len(o) } func (o treeItemReqByIRI) Less(i, j int) bool { return o[i].ID < o[j].ID } func (o treeItemReqByIRI) Swap(i, j int) { o[i], o[j] = o[j], o[i] } type treeItemReq struct { ID quad.IRI `quad:"@id"` Name string `quad:"name"` Children []treeItemReq `quad:"child,required"` } func (t *treeItemReq) Sort() { for _, c := range t.Children { c.Sort() } sort.Sort(treeItemReqByIRI(t.Children)) } type genObject struct { ID quad.IRI `quad:"@id"` Name string `quad:"name"` } type MyString string type genObjectTypedef struct { ID quad.IRI `quad:"@id"` Name MyString `quad:"name"` } type subObject struct { genObject Num int `quad:"num"` } type subSubObject struct { subObject Num2 int `quad:"num2"` } type Coords struct { Lat float64 `json:"ex:lat"` Lng float64 `json:"ex:lng"` } type NodeLoop struct { ID quad.IRI `quad:"@id"` Name string `quad:"name"` Next *NodeLoop `quad:"next"` } type NestedNode struct { ID quad.IRI `quad:"@id"` Name string `quad:"name"` Prev genObject `quad:"prev,opt"` Next genObject `quad:"next,opt"` } type Alts struct { Alt []OptFields `quad:"alt"` } type OptFields struct { One string `quad:"one,optional"` Two string `quad:"two,optional"` } func iri(s string) quad.IRI { return quad.IRI(s) } const typeIRI = quad.IRI(rdf.Type) var treeQuads = []quad.Quad{ {iri("n1"), iri("name"), quad.String("Node 1"), nil}, {iri("n2"), iri("name"), quad.String("Node 2"), nil}, {iri("n3"), iri("name"), quad.String("Node 3"), nil}, {iri("n4"), iri("name"), quad.String("Node 4"), nil}, {iri("n5"), iri("name"), quad.String("Node 5"), nil}, {iri("n1"), iri("child"), iri("n2"), nil}, {iri("n1"), iri("child"), iri("n3"), nil}, {iri("n3"), iri("child"), iri("n4"), nil}, } ================================================ FILE: schema/types.go ================================================ package schema import ( "github.com/cayleygraph/quad" _ "github.com/cayleygraph/quad/voc/rdf" _ "github.com/cayleygraph/quad/voc/rdfs" "github.com/cayleygraph/quad/voc/schema" ) func init() { RegisterType(quad.IRI(schema.Class), Class{}) RegisterType(quad.IRI(schema.Property), Property{}) } // Resource is a basic RDF resource object. type Resource struct { ID quad.IRI `quad:"@id"` Label string `quad:"rdfs:label,optional"` Comment string `quad:"rdfs:comment,optional"` Name string `quad:"schema:name,optional"` Description string `quad:"schema:description,optional"` } type Property struct { Resource InverseOf quad.IRI `quad:"schema:inverseOf,optional"` SupersededBy []quad.IRI `quad:"schema:supersededBy,optional"` Expects []quad.IRI `quad:"schema:rangeIncludes"` } type Class struct { Resource Properties []Property `quad:"schema:domainIncludes < *,optional"` SupersededBy []quad.IRI `quad:"schema:supersededBy,optional"` Extends []quad.IRI `quad:"rdfs:subClassOf,optional"` } type PropertiesByIRI []Property func (a PropertiesByIRI) Len() int { return len(a) } func (a PropertiesByIRI) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a PropertiesByIRI) Less(i, j int) bool { return a[i].ID < a[j].ID } type ClassesByIRI []Class func (a ClassesByIRI) Len() int { return len(a) } func (a ClassesByIRI) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ClassesByIRI) Less(i, j int) bool { return a[i].ID < a[j].ID } ================================================ FILE: schema/writer.go ================================================ package schema import ( "fmt" "reflect" "github.com/cayleygraph/quad" ) // WriteAsQuads writes a single value in form of quads into specified quad writer. // // It returns an identifier of the object in the output sub-graph. If an object has // an annotated ID field, it's value will be converted to quad.Value and returned. // Otherwise, a new BNode will be generated using GenerateID function. // // See LoadTo for a list of quads mapping rules. func (c *Config) WriteAsQuads(w quad.Writer, o interface{}) (quad.Value, error) { wr := c.newWriter(w) return wr.writeAsQuads(reflect.ValueOf(o)) } type writer struct { c *Config w quad.Writer seen map[uintptr]quad.Value } func (c *Config) newWriter(w quad.Writer) *writer { return &writer{c: c, w: w, seen: make(map[uintptr]quad.Value)} } func (w *writer) writeQuad(s, p, o quad.Value, rev bool) error { if rev { s, o = o, s } return w.w.WriteQuad(quad.Quad{Subject: s, Predicate: p, Object: o, Label: w.c.Label}) } // writeOneValReflect writes a set of quads corresponding to a value. It may omit writing quads if value is zero. func (w *writer) writeOneValReflect(id quad.Value, pred quad.Value, rv reflect.Value, rev bool) error { if isZero(rv) { return nil } // write field value and get an ID sid, err := w.writeAsQuads(rv) if err != nil { return err } // write a quad pointing to this value return w.writeQuad(id, pred, sid, rev) } func (w *writer) writeTypeInfo(id quad.Value, rt reflect.Type) error { iri := getTypeIRI(rt) if iri == quad.IRI("") { return nil } return w.writeQuad(id, w.c.iri(iriType), w.c.iri(iri), false) } func (w *writer) writeValueAs(id quad.Value, rv reflect.Value, pref string, rules fieldRules) error { switch kind := rv.Kind(); kind { case reflect.Ptr, reflect.Map: ptr := rv.Pointer() if _, ok := w.seen[ptr]; ok { return nil } w.seen[ptr] = id if kind == reflect.Ptr { rv = rv.Elem() } } rt := rv.Type() if err := w.writeTypeInfo(id, rt); err != nil { return err } for i := 0; i < rt.NumField(); i++ { f := rt.Field(i) if f.Anonymous { if err := w.writeValueAs(id, rv.Field(i), pref+f.Name+".", rules); err != nil { return err } continue } switch r := rules[pref+f.Name].(type) { case constraintRule: s, o := id, quad.Value(r.Val) if r.Rev { s, o = o, s } if err := w.writeQuad(s, r.Pred, o, false); err != nil { return err } case saveRule: if f.Type.Kind() == reflect.Slice { sl := rv.Field(i) for j := 0; j < sl.Len(); j++ { if err := w.writeOneValReflect(id, r.Pred, sl.Index(j), r.Rev); err != nil { return err } } } else { fv := rv.Field(i) if !r.Opt && isZero(fv) { return ErrReqFieldNotSet{Field: f.Name} } if err := w.writeOneValReflect(id, r.Pred, fv, r.Rev); err != nil { return err } } } } return nil } func (w *writer) writeAsQuads(rv reflect.Value) (quad.Value, error) { rt := rv.Type() // if node is a primitive - return directly if rt.Implements(reflQuadValue) { return rv.Interface().(quad.Value), nil } prv := rv kind := rt.Kind() // check if we've seen this node already switch kind { case reflect.Ptr, reflect.Map: ptr := prv.Pointer() if sid, ok := w.seen[ptr]; ok { return sid, nil } if kind == reflect.Ptr { rv = rv.Elem() rt = rv.Type() kind = rt.Kind() } } // check if it's a type that quads package supports // note, that it may be a struct such as time.Time if val, ok := quad.AsValue(rv.Interface()); ok { return val, nil } else if kind == reflect.String { return quad.String(rv.String()), nil } else if kind == reflect.Int || kind == reflect.Uint || kind == reflect.Int32 || kind == reflect.Uint32 || kind == reflect.Int16 || kind == reflect.Uint16 || kind == reflect.Int8 || kind == reflect.Uint8 { return quad.Int(rv.Int()), nil } else if kind == reflect.Float64 || kind == reflect.Float32 { return quad.Float(rv.Float()), nil } else if kind == reflect.Bool { return quad.Bool(rv.Bool()), nil } // TODO(dennwc): support maps if kind != reflect.Struct { return nil, fmt.Errorf("unsupported type: %v", rt) } // get conversion rules for this struct type rules, err := w.c.rulesFor(rt) if err != nil { return nil, fmt.Errorf("can't load rules: %v", err) } if len(rules) == 0 { return nil, fmt.Errorf("no rules for struct: %v", rt) } // get an ID from the struct value id, err := w.c.idFor(rules, rt, rv, "") if err != nil { return nil, err } if id == nil { id = w.c.genID(prv.Interface()) } // save a node ID to avoid loops switch prv.Kind() { case reflect.Ptr, reflect.Map: ptr := prv.Pointer() w.seen[ptr] = id } if err = w.writeValueAs(id, rv, "", rules); err != nil { return nil, err } return id, nil } ================================================ FILE: schema/writer_test.go ================================================ package schema_test import ( "reflect" "testing" "time" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" ) type quadSlice []quad.Quad func (s *quadSlice) WriteQuad(q quad.Quad) error { *s = append(*s, q) return nil } func (s *quadSlice) WriteQuads(buf []quad.Quad) (int, error) { *s = append(*s, buf...) return len(buf), nil } func TestWriteAsQuads(t *testing.T) { sch := schema.NewConfig() for _, c := range testWriteValueCases { t.Run(c.name, func(t *testing.T) { var out quadSlice id, err := sch.WriteAsQuads(&out, c.obj) if err != c.err { t.Errorf("unexpected error: %v (expected: %v)", err, c.err) } else if c.err != nil { return // case with expected error; omit other checks } if c.id == nil { for i := range out { if c.expect[i].Subject == nil { c.expect[i].Subject = id } } } else if id != c.id { t.Errorf("ids are different: %v vs %v", id, c.id) } if !reflect.DeepEqual([]quad.Quad(out), c.expect) { t.Errorf("quad sets are different\n%#v\n%#v", []quad.Quad(out), c.expect) } }) } } var testWriteValueCases = []struct { name string obj interface{} id quad.Value expect []quad.Quad err error }{ { "complex object", struct { rdfType struct{} `quad:"rdf:type > some:Type"` ID quad.IRI `quad:"@id"` Name string `quad:"name"` Values []string `quad:"values"` Items []item `quad:"items"` Sub *item `quad:"sub"` }{ ID: "1234", Name: "some item", Values: []string{"val1", "val2"}, Items: []item{ {ID: "sub1", Name: "Sub 1"}, {ID: "sub2", Name: "Sub 2"}, }, Sub: &item{ID: "sub3", Name: "Sub 3"}, }, iri("1234"), []quad.Quad{ {iri("1234"), typeIRI, iri("some:Type"), nil}, {iri("1234"), iri("name"), quad.String(`some item`), nil}, {iri("1234"), iri("values"), quad.String(`val1`), nil}, {iri("1234"), iri("values"), quad.String(`val2`), nil}, {iri("sub1"), typeIRI, iri("some:item"), nil}, {iri("sub1"), iri("name"), quad.String(`Sub 1`), nil}, {iri("1234"), iri("items"), iri("sub1"), nil}, {iri("sub2"), typeIRI, iri("some:item"), nil}, {iri("sub2"), iri("name"), quad.String(`Sub 2`), nil}, {iri("1234"), iri("items"), iri("sub2"), nil}, {iri("sub3"), typeIRI, iri("some:item"), nil}, {iri("sub3"), iri("name"), quad.String(`Sub 3`), nil}, {iri("1234"), iri("sub"), iri("sub3"), nil}, }, nil, }, { "complex object (embedded)", struct { rdfType struct{} `quad:"rdf:type > some:Type"` item2 ID quad.IRI `quad:"@id"` Values []string `quad:"values"` }{ item2: item2{Name: "Sub 1", Spec: "special"}, ID: "1234", Values: []string{"val1", "val2"}, }, iri("1234"), []quad.Quad{ {iri("1234"), typeIRI, iri("some:Type"), nil}, {iri("1234"), iri("name"), quad.String(`Sub 1`), nil}, {iri("1234"), iri("spec"), quad.String(`special`), nil}, {iri("1234"), iri("values"), quad.String(`val1`), nil}, {iri("1234"), iri("values"), quad.String(`val2`), nil}, }, nil, }, { "type shorthand", struct { rdfType struct{} `quad:"@type > some:Type"` item2 ID quad.IRI `quad:"@id"` Values []string `quad:"values"` }{ item2: item2{Name: "Sub 1", Spec: "special"}, ID: "1234", Values: []string{"val1", "val2"}, }, iri("1234"), []quad.Quad{ {iri("1234"), typeIRI, iri("some:Type"), nil}, {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, {iri("1234"), iri("spec"), quad.String("special"), nil}, {iri("1234"), iri("values"), quad.String("val1"), nil}, {iri("1234"), iri("values"), quad.String("val2"), nil}, }, nil, }, { "json tags", struct { rdfType struct{} `quad:"@type > some:Type"` item2 ID quad.IRI `json:"@id"` Values []string `json:"values,omitempty"` }{ item2: item2{Name: "Sub 1", Spec: "special"}, ID: "1234", Values: []string{"val1", "val2"}, }, iri("1234"), []quad.Quad{ {iri("1234"), typeIRI, iri("some:Type"), nil}, {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, {iri("1234"), iri("spec"), quad.String("special"), nil}, {iri("1234"), iri("values"), quad.String("val1"), nil}, {iri("1234"), iri("values"), quad.String("val2"), nil}, }, nil, }, { "simple object", subObject{ genObject: genObject{ ID: "1234", Name: "Obj", }, Num: 3, }, iri("1234"), []quad.Quad{ {iri("1234"), iri("name"), quad.String("Obj"), nil}, {iri("1234"), iri("num"), quad.Int(3), nil}, }, nil, }, { "typedef", genObjectTypedef{ ID: "1234", Name: "Obj", }, iri("1234"), []quad.Quad{ {iri("1234"), iri("name"), quad.String("Obj"), nil}, }, nil, }, { "simple object (embedded multiple levels)", subSubObject{ subObject: subObject{ genObject: genObject{ ID: "1234", Name: "Obj", }, Num: 3, }, Num2: 4, }, iri("1234"), []quad.Quad{ {iri("1234"), iri("name"), quad.String("Obj"), nil}, {iri("1234"), iri("num"), quad.Int(3), nil}, {iri("1234"), iri("num2"), quad.Int(4), nil}, }, nil, }, { "required field not set", item2{Name: "partial"}, nil, nil, schema.ErrReqFieldNotSet{Field: "Spec"}, }, { "required unexported", timeItem{ID: "1", Name: "t", TS: time.Unix(100, 0)}, nil, []quad.Quad{ {iri("1"), iri("name"), quad.String("t"), nil}, {iri("1"), iri("ts"), quad.Time(time.Unix(100, 0)), nil}, }, nil, }, { "single tree node", treeItemOpt{ ID: iri("n1"), Name: "Node 1", }, iri("n1"), []quad.Quad{ {iri("n1"), iri("name"), quad.String("Node 1"), nil}, }, nil, }, { "nested tree nodes", treeItemOpt{ ID: iri("n1"), Name: "Node 1", Children: []treeItemOpt{ {ID: iri("n2"), Name: "Node 2"}, }, }, iri("n1"), []quad.Quad{ {iri("n1"), iri("name"), quad.String("Node 1"), nil}, {iri("n2"), iri("name"), quad.String("Node 2"), nil}, {iri("n1"), iri("child"), iri("n2"), nil}, }, nil, }, { "coords", Coords{Lat: 12.3, Lng: 34.5}, nil, []quad.Quad{ {nil, typeIRI, iri("ex:Coords"), nil}, {nil, iri("ex:lat"), quad.Float(12.3), nil}, {nil, iri("ex:lng"), quad.Float(34.5), nil}, }, nil, }, { "self loop", func() *NodeLoop { a := &NodeLoop{ID: iri("A"), Name: "Node A"} a.Next = a return a }(), iri("A"), []quad.Quad{ {iri("A"), iri("name"), quad.String("Node A"), nil}, {iri("A"), iri("next"), iri("A"), nil}, }, nil, }, { "pointer chain", func() *NodeLoop { a := &NodeLoop{ID: iri("A"), Name: "Node A"} b := &NodeLoop{ID: iri("B"), Name: "Node B"} c := &NodeLoop{ID: iri("C"), Name: "Node C"} a.Next = b b.Next = c c.Next = a return a }(), iri("A"), []quad.Quad{ {iri("A"), iri("name"), quad.String("Node A"), nil}, {iri("B"), iri("name"), quad.String("Node B"), nil}, {iri("C"), iri("name"), quad.String("Node C"), nil}, {iri("C"), iri("next"), iri("A"), nil}, {iri("B"), iri("next"), iri("C"), nil}, {iri("A"), iri("next"), iri("B"), nil}, }, nil, }, } ================================================ FILE: server/http/accept.go ================================================ // Copyright 2013 The Go Authors. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd. package cayleyhttp import ( "net/http" "strings" ) // Octet types from RFC 2616. var octetTypes [256]octetType type octetType byte const ( isToken octetType = 1 << iota isSpace ) func init() { // OCTET = // CHAR = // CTL = // CR = // LF = // SP = // HT = // <"> = // CRLF = CR LF // LWS = [CRLF] 1*( SP | HT ) // TEXT = // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT // token = 1* // qdtext = > for c := 0; c < 256; c++ { var t octetType isCtl := c <= 31 || c == 127 isChar := 0 <= c && c <= 127 isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { t |= isSpace } if isChar && !isCtl && !isSeparator { t |= isToken } octetTypes[c] = t } } // AcceptSpec describes an Accept* header. type AcceptSpec struct { Value string Q float64 } // ParseAccept parses Accept* headers. func ParseAccept(header http.Header, key string) (specs []AcceptSpec) { loop: for _, s := range header[key] { for { var spec AcceptSpec spec.Value, s = expectTokenSlash(s) if spec.Value == "" { continue loop } spec.Q = 1.0 s = skipSpace(s) if strings.HasPrefix(s, ";") { s = skipSpace(s[1:]) if !strings.HasPrefix(s, "q=") { continue loop } spec.Q, s = expectQuality(s[2:]) if spec.Q < 0.0 { continue loop } } specs = append(specs, spec) s = skipSpace(s) if !strings.HasPrefix(s, ",") { continue loop } s = skipSpace(s[1:]) } } return } func skipSpace(s string) (rest string) { i := 0 for ; i < len(s); i++ { if octetTypes[s[i]]&isSpace == 0 { break } } return s[i:] } func expectTokenSlash(s string) (token, rest string) { i := 0 for ; i < len(s); i++ { b := s[i] if (octetTypes[b]&isToken == 0) && b != '/' { break } } return s[:i], s[i:] } func expectQuality(s string) (q float64, rest string) { switch { case len(s) == 0: return -1, "" case s[0] == '0': q = 0 case s[0] == '1': q = 1 default: return -1, "" } s = s[1:] if !strings.HasPrefix(s, ".") { return q, s } s = s[1:] i := 0 n := 0 d := 1 for ; i < len(s); i++ { b := s[i] if b < '0' || b > '9' { break } n = n*10 + int(b) - '0' d *= 10 } return q + float64(n)/float64(d), s[i:] } ================================================ FILE: server/http/api_v2.go ================================================ // Copyright 2017 The Cayley Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cayleyhttp import ( "compress/gzip" "context" "encoding/json" "errors" "fmt" "io" "io/ioutil" "net/http" "strings" "time" "github.com/julienschmidt/httprouter" "github.com/cayleygraph/cayley/clog" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/query" "github.com/cayleygraph/cayley/query/shape" // Writer is imported for writers to be registered _ "github.com/cayleygraph/cayley/writer" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/voc" ) const ( prefix = "/api/v2" defaultLimit = 100 defaultReplication = "single" ) // NewAPIv2 creates a new instance of APIv2 with default options func NewAPIv2(h *graph.Handle, wrappers ...HandlerWrapper) *APIv2 { return NewAPIv2Writer(h, defaultReplication, nil, wrappers...) } // NewBoundAPIv2 creates a new instance of APIv2 bound to a given httprouter.Router func NewBoundAPIv2(h *graph.Handle, r *httprouter.Router) *APIv2 { api := &APIv2{h: h, wtyp: defaultReplication, wopt: nil, limit: defaultLimit, handler: r} api.registerOn(r) return api } // NewAPIv2Writer creates a new instance of APIv2 func NewAPIv2Writer(h *graph.Handle, wtype string, wopts graph.Options, wrappers ...HandlerWrapper) *APIv2 { r := httprouter.New() api := &APIv2{h: h, wtyp: wtype, wopt: wopts, limit: defaultLimit} api.registerOn(r) var handler http.Handler = r for _, wrapper := range wrappers { handler = wrapper(handler) } api.handler = handler return api } // APIv2 holds state and configuration of a request type APIv2 struct { h *graph.Handle ro bool batch int handler http.Handler // replication wtyp string wopt graph.Options // query timeout time.Duration limit int } // SetReadOnly sets read-only mode for the request func (api *APIv2) SetReadOnly(ro bool) { api.ro = ro } // SetBatchSize sets batch-size mode for the request func (api *APIv2) SetBatchSize(n int) { api.batch = n } // SetQueryTimeout sets query timeout for the request func (api *APIv2) SetQueryTimeout(dt time.Duration) { api.timeout = dt } // SetQueryLimit sets query limit for the request func (api *APIv2) SetQueryLimit(n int) { api.limit = n } // ServeHTTP implements http.Handler func (api *APIv2) ServeHTTP(w http.ResponseWriter, r *http.Request) { api.handler.ServeHTTP(w, r) } // HandlerWrapper accepts a handler, wraps it with additional functionality and // returns a new handler type HandlerWrapper func(http.Handler) http.Handler // toHandle wraps a http.HandlerFunc to comply with the signature of httprouter.Handle func toHandle(handler http.HandlerFunc) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { handler(w, r) } } func (api *APIv2) registerDataOn(r *httprouter.Router) { if !api.ro { r.POST(prefix+"/write", toHandle(api.ServeWrite)) r.POST(prefix+"/delete", toHandle(api.ServeDelete)) r.POST(prefix+"/node/delete", toHandle(api.ServeNodeDelete)) } r.POST(prefix+"/read", toHandle(api.ServeRead)) r.GET(prefix+"/read", toHandle(api.ServeRead)) r.GET(prefix+"/formats", toHandle(api.ServeFormats)) } func (api *APIv2) registerQueryOn(r *httprouter.Router) { r.POST(prefix+"/query", toHandle(api.ServeQuery)) r.GET(prefix+"/query", toHandle(api.ServeQuery)) } func (api *APIv2) registerOn(r *httprouter.Router) { api.registerDataOn(r) api.registerQueryOn(r) } const ( defaultFormat = "nquads" hdrContentType = "Content-Type" hdrContentEncoding = "Content-Encoding" hdrAccept = "Accept" hdrAcceptEncoding = "Accept-Encoding" contentTypeJSON = "application/json" contentTypeJSONLD = "application/ld+json" ) func getFormat(r *http.Request, formKey string, acceptName string) *quad.Format { var format *quad.Format if formKey != "" { if name := r.FormValue("format"); name != "" { format = quad.FormatByName(name) } } if acceptName != "" && format == nil { specs := ParseAccept(r.Header, acceptName) // TODO: sort by Q if len(specs) != 0 { format = quad.FormatByMime(specs[0].Value) } } if format == nil { format = quad.FormatByName(defaultFormat) } return format } func readerFrom(r *http.Request, acceptName string) (io.ReadCloser, error) { if specs := ParseAccept(r.Header, acceptName); len(specs) != 0 { if s := specs[0]; s.Value == "gzip" { zr, err := gzip.NewReader(r.Body) if err != nil { return nil, err } return zr, nil } } return r.Body, nil } type nopWriteCloser struct { io.Writer } func (nopWriteCloser) Close() error { return nil } func writerFrom(w http.ResponseWriter, r *http.Request, acceptName string) io.WriteCloser { if specs := ParseAccept(r.Header, acceptName); len(specs) != 0 { if s := specs[0]; s.Value == "gzip" { w.Header().Set(hdrContentEncoding, s.Value) zw := gzip.NewWriter(w) return zw } } return nopWriteCloser{Writer: w} } func (api *APIv2) handleForRequest(r *http.Request) (*graph.Handle, error) { return HandleForRequest(api.h, api.wtyp, api.wopt, r) } // writeResponse represents the response received for a successful write type writeResponse struct { Result string `json:"result"` Count int `json:"count"` } // newWriteResponse creates a new WriteResponse for given count of quads written func newWriteResponse(count int) writeResponse { return writeResponse{ Result: fmt.Sprintf("Successfully wrote %d quads.", count), Count: count, } } // ServeWrite writes data received in the request body to the database func (api *APIv2) ServeWrite(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() if api.ro { jsonResponse(w, http.StatusForbidden, errors.New("database is read-only")) return } format := getFormat(r, "", hdrContentType) if format == nil || format.Reader == nil { jsonResponse(w, http.StatusBadRequest, errors.New("format is not supported for reading data")) return } rd, err := readerFrom(r, hdrContentEncoding) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } defer rd.Close() qr := format.Reader(rd) defer qr.Close() h, err := api.handleForRequest(r) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } qw := graph.NewWriter(h.QuadWriter) defer qw.Close() n, err := quad.CopyBatch(qw, qr, api.batch) if err != nil { jsonResponse(w, http.StatusInternalServerError, err) return } err = qw.Close() if err != nil { jsonResponse(w, http.StatusInternalServerError, err) return } w.Header().Set(hdrContentType, contentTypeJSON) response := newWriteResponse(n) encoder := json.NewEncoder(w) encoder.Encode(response) } // ServeDelete deletes data received in the request body from the database. // Responds with how many quads were deleted. func (api *APIv2) ServeDelete(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() if api.ro { jsonResponse(w, http.StatusForbidden, errors.New("database is read-only")) return } format := getFormat(r, "", hdrContentType) if format == nil || format.Reader == nil { jsonResponse(w, http.StatusBadRequest, fmt.Errorf("format is not supported for reading quads")) return } rd, err := readerFrom(r, hdrContentEncoding) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } defer rd.Close() qr := format.Reader(r.Body) defer qr.Close() h, err := api.handleForRequest(r) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } qw := graph.NewRemover(h.QuadWriter) defer qw.Close() n, err := quad.CopyBatch(qw, qr, api.batch) if err != nil { jsonResponse(w, http.StatusInternalServerError, err) return } w.Header().Set(hdrContentType, contentTypeJSON) fmt.Fprintf(w, `{"result": "Successfully deleted %d quads.", "count": %d}`+"\n", n, n) } // ServeNodeDelete deletes all data associated with a node (an entity). // Responds with how many quads were deleted. func (api *APIv2) ServeNodeDelete(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() if api.ro { jsonResponse(w, http.StatusForbidden, errors.New("database is read-only")) return } format := getFormat(r, "", hdrContentType) if format == nil || format.UnmarshalValue == nil { jsonResponse(w, http.StatusBadRequest, fmt.Errorf("format is not supported for reading nodes")) return } const limit = 128*1024 + 1 rd := io.LimitReader(r.Body, limit) data, err := ioutil.ReadAll(rd) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } else if len(data) == limit { jsonResponse(w, http.StatusBadRequest, fmt.Errorf("request data is too large")) return } v, err := format.UnmarshalValue(data) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } else if v == nil { jsonResponse(w, http.StatusBadRequest, fmt.Errorf("cannot remove nil value")) return } h, err := api.handleForRequest(r) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } err = h.RemoveNode(v) if err != nil { jsonResponse(w, http.StatusInternalServerError, err) return } w.Header().Set(hdrContentType, contentTypeJSON) const n = 1 fmt.Fprintf(w, `{"result": "Successfully deleted %d nodes.", "count": %d}`+"\n", n, n) } type checkWriter struct { w io.Writer written bool } func (w *checkWriter) Write(p []byte) (int, error) { w.written = true return w.w.Write(p) } func valuesFromString(s string) []quad.Value { if s == "" { return nil } arr := strings.Split(s, ",") out := make([]quad.Value, 0, len(arr)) for _, s := range arr { out = append(out, quad.StringToValue(s)) } return out } // ServeRead responds with quads read from the database func (api *APIv2) ServeRead(w http.ResponseWriter, r *http.Request) { format := getFormat(r, "format", hdrAccept) if format == nil || format.Writer == nil { jsonResponse(w, http.StatusBadRequest, fmt.Errorf("format is not supported for reading data")) return } h, err := api.handleForRequest(r) if err != nil { jsonResponse(w, http.StatusBadRequest, err) return } values := shape.FilterQuads( valuesFromString(r.FormValue("sub")), valuesFromString(r.FormValue("pred")), valuesFromString(r.FormValue("obj")), valuesFromString(r.FormValue("label")), ) it := values.BuildIterator(h.QuadStore).Iterate() qr := graph.NewResultReader(h.QuadStore, it) defer qr.Close() wr := writerFrom(w, r, hdrAcceptEncoding) defer wr.Close() cw := &checkWriter{w: wr} qwc := format.Writer(cw) defer qwc.Close() var qw quad.Writer = qwc if len(format.Mime) != 0 { w.Header().Set(hdrContentType, format.Mime[0]) } if irif := r.FormValue("iri"); irif != "" { opts := quad.IRIOptions{ Format: quad.IRIDefault, } switch irif { case "short": opts.Format = quad.IRIShort case "full": opts.Format = quad.IRIFull } qw = quad.IRIWriter(qw, opts) } if bw, ok := qw.(quad.BatchWriter); ok { _, err = quad.CopyBatch(bw, qr, api.batch) } else { _, err = quad.Copy(qw, qr) } if err != nil && !cw.written { jsonResponse(w, http.StatusInternalServerError, err) return } else if err != nil { // can do nothing here, since first byte (and header) was written // TODO: check if client just gone away clog.Errorf("read quads error: %v", err) } } // ServeFormats responds with formats known for the database func (api *APIv2) ServeFormats(w http.ResponseWriter, r *http.Request) { type Format struct { ID string `json:"id"` Read bool `json:"read,omitempty"` Write bool `json:"write,omitempty"` Nodes bool `json:"nodes,omitempty"` Ext []string `json:"ext,omitempty"` Mime []string `json:"mime,omitempty"` Binary bool `json:"binary,omitempty"` } formats := quad.Formats() out := make([]Format, 0, len(formats)) for _, f := range formats { out = append(out, Format{ ID: f.Name, Ext: f.Ext, Mime: f.Mime, Read: f.Reader != nil, Write: f.Writer != nil, Nodes: f.UnmarshalValue != nil, Binary: f.Binary, }) } w.Header().Set(hdrContentType, contentTypeJSON) json.NewEncoder(w).Encode(out) } func (api *APIv2) queryContext(r *http.Request) (ctx context.Context, cancel func()) { ctx = r.Context() if api.timeout > 0 { ctx, cancel = context.WithTimeout(ctx, api.timeout) } else { ctx, cancel = context.WithCancel(ctx) } return ctx, cancel } func defaultErrorFunc(w query.ResponseWriter, err error) { data, _ := json.Marshal(err.Error()) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(`{"error": `)) w.Write(data) w.Write([]byte("}\n")) } func writeResults(w io.Writer, r interface{}) { enc := json.NewEncoder(w) enc.SetEscapeHTML(false) enc.Encode(map[string]interface{}{ "result": r, }) } const maxQuerySize = 1024 * 1024 // 1 MB func readLimit(r io.Reader) ([]byte, error) { lr := io.LimitReader(r, maxQuerySize).(*io.LimitedReader) data, err := ioutil.ReadAll(lr) if err != nil && lr.N <= 0 { err = errors.New("request is too large") } return data, err } // ServeQuery executes a query received in the request and responds with the result func (api *APIv2) ServeQuery(w http.ResponseWriter, r *http.Request) { ctx, cancel := api.queryContext(r) defer cancel() vals := r.URL.Query() lang := vals.Get("lang") if lang == "" { jsonResponse(w, http.StatusBadRequest, "query language not specified") return } l := query.GetLanguage(lang) if l == nil { jsonResponse(w, http.StatusBadRequest, "unknown query language") return } errFunc := defaultErrorFunc if l.HTTPError != nil { errFunc = l.HTTPError } select { case <-ctx.Done(): errFunc(w, ctx.Err()) return default: } h, err := api.handleForRequest(r) if err != nil { errFunc(w, err) return } if l.HTTPQuery != nil { defer r.Body.Close() l.HTTPQuery(ctx, h.QuadStore, w, r.Body) return } if l.Session == nil { errFunc(w, errors.New("HTTP interface is not supported for this query language")) return } ses := l.Session(h.QuadStore) var qu string if r.Method == "GET" { qu = vals.Get("qu") } else { data, err := readLimit(r.Body) if err != nil { errFunc(w, err) return } qu = string(data) } if qu == "" { jsonResponse(w, http.StatusBadRequest, "query is empty") return } if clog.V(1) { clog.Infof("query: %s: %q", lang, qu) } opt := query.Options{ Collation: query.JSON, // TODO: switch to JSON-LD by default when the time comes Limit: api.limit, } if specs := ParseAccept(r.Header, hdrAccept); len(specs) != 0 { // TODO: sort by Q switch specs[0].Value { case contentTypeJSON: opt.Collation = query.JSON case contentTypeJSONLD: opt.Collation = query.JSONLD } } it, err := ses.Execute(ctx, qu, opt) if err != nil { errFunc(w, err) return } defer it.Close() var out []interface{} for it.Next(ctx) { out = append(out, it.Result()) } if err = it.Err(); err != nil { errFunc(w, err) return } if opt.Collation == query.JSONLD { w.Header().Set(hdrContentType, contentTypeJSONLD) } else { w.Header().Set(hdrContentType, contentTypeJSON) } writeResults(w, out) } // NamespaceRule defines a prefix for a namespace when prepended to the suffix of a compact IRI, results in an IRI. // For example rdfs is a prefix for the namespace http://www.w3.org/2000/01/rdf-schema#. type NamespaceRule struct { Prefix string `json:"prefix"` Namespace string `json:"namespace"` } // getNamespaceRules returns all the registered rules func getNamespaceRules() []NamespaceRule { var rules []NamespaceRule for _, n := range voc.List() { rules = append(rules, NamespaceRule{ Prefix: strings.TrimSuffix(n.Prefix, ":"), Namespace: n.Full, }) } return rules } // serveGetNamespaceRules responds with all the registered rules encoded to JSON func serveGetNamespaceRules(w http.ResponseWriter) { rules := getNamespaceRules() encoder := json.NewEncoder(w) w.Header().Set(hdrContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) encoder.Encode(rules) } // serveGetNamespaceRules registers received rule encoded in JSON func servePostNamespaceRules(w http.ResponseWriter, r *http.Request) { var rule NamespaceRule decoder := json.NewDecoder(r.Body) decoder.Decode(&rule) voc.RegisterPrefix(rule.Prefix, rule.Namespace) w.WriteHeader(http.StatusCreated) } // ServeNamespaceRules serves requests for the namespace rules resource. // The resource supports getting all registered rules and registering a rule. // The resource wraps the quad/voc module. func (api *APIv2) ServeNamespaceRules(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: serveGetNamespaceRules(w) return case http.MethodPost: servePostNamespaceRules(w, r) return default: jsonResponse(w, http.StatusMethodNotAllowed, nil) } } ================================================ FILE: server/http/api_v2_test.go ================================================ package cayleyhttp import ( "bytes" "encoding/json" "io" "net/http" "net/http/httptest" "sort" "testing" "github.com/cayleygraph/cayley/graph" "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/writer" "github.com/cayleygraph/quad" "github.com/cayleygraph/quad/jsonld" "github.com/stretchr/testify/require" ) func makeHandle(t testing.TB, quads ...quad.Quad) *graph.Handle { qs := memstore.New(quads...) wr, err := writer.NewSingleReplication(qs, nil) require.NoError(t, err) return &graph.Handle{ QuadStore: qs, QuadWriter: wr, } } func makeServerV2(t testing.TB, quads ...quad.Quad) *APIv2 { h := makeHandle(t, quads...) return NewAPIv2(h) } func writeQuads(q []quad.Quad, w io.Writer) error { writer := jsonld.NewWriter(w) reader := quad.NewReader(quads) _, err := quad.Copy(writer, reader) writer.Close() return err } func newQuadsBuffer(quads []quad.Quad) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) err := writeQuads(quads, buf) return buf, err } var mime = quad.FormatByName("jsonld").Mime[0] var quads = []quad.Quad{ quad.MakeIRI("http://example.com/bob", "http://example.com/likes", "http://example.com/alice", ""), quad.MakeIRI("http://example.com/alice", "http://example.com/likes", "http://example.com/bob", ""), } func TestV2Write(t *testing.T) { api := makeServerV2(t) buf, err := newQuadsBuffer(quads) req, err := http.NewRequest(http.MethodGet, prefix+"/write", buf) require.NoError(t, err) req.Header.Set(hdrContentType, mime) rr := httptest.NewRecorder() handler := http.HandlerFunc(api.ServeWrite) handler.ServeHTTP(rr, req) require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) expectedResponse := newWriteResponse(len(quads)) var response writeResponse json.Unmarshal(rr.Body.Bytes(), &response) require.Equal(t, expectedResponse, response) } func TestV2Read(t *testing.T) { api := makeServerV2(t, quads...) buf := bytes.NewBuffer(nil) req, err := http.NewRequest(http.MethodGet, prefix+"/read", buf) require.NoError(t, err) req.Header.Set(hdrAccept, mime) rr := httptest.NewRecorder() handler := http.HandlerFunc(api.ServeRead) handler.ServeHTTP(rr, req) require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) reader := jsonld.NewReader(rr.Body) receivedQuads, err := quad.ReadAll(reader) require.NoError(t, err) sort.Sort(quad.ByQuadString(receivedQuads)) sort.Sort(quad.ByQuadString(quads)) require.Equal(t, quads, receivedQuads) } func TestV2Delete(t *testing.T) { api := makeServerV2(t, quads...) buf, err := newQuadsBuffer(quads) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, prefix+"/delete", buf) require.NoError(t, err) req.Header.Set(hdrContentType, mime) rr := httptest.NewRecorder() handler := http.HandlerFunc(api.ServeDelete) handler.ServeHTTP(rr, req) require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) } func TestV2GetNamespaceRules(t *testing.T) { api := makeServerV2(t) buf := bytes.NewBuffer(nil) req, err := http.NewRequest(http.MethodGet, prefix+"/namespace-rules", buf) require.NoError(t, err) rr := httptest.NewRecorder() handler := http.HandlerFunc(api.ServeNamespaceRules) handler.ServeHTTP(rr, req) var rules []NamespaceRule err = json.Unmarshal(rr.Body.Bytes(), &rules) require.NoError(t, err) require.Equal(t, http.StatusOK, rr.Code) require.Equal(t, contentTypeJSON, rr.Header().Get(hdrContentType)) require.NotEmpty(t, rules) } func TestV2PostNamespaceRules(t *testing.T) { api := makeServerV2(t) rule := NamespaceRule{ Prefix: "foaf", Namespace: "http://xmlns.com/foaf/0.1/", } buf := bytes.NewBuffer(nil) encoder := json.NewEncoder(buf) encoder.Encode(rule) req, err := http.NewRequest(http.MethodPost, prefix+"/namespace-rules", buf) require.NoError(t, err) rr := httptest.NewRecorder() handler := http.HandlerFunc(api.ServeNamespaceRules) handler.ServeHTTP(rr, req) require.NoError(t, err) require.Equal(t, http.StatusCreated, rr.Code) require.Empty(t, rr.Body.Bytes()) // Check effect req, err = http.NewRequest(http.MethodGet, prefix+"/namespace-rules", buf) require.NoError(t, err) rr = httptest.NewRecorder() handler = http.HandlerFunc(api.ServeNamespaceRules) handler.ServeHTTP(rr, req) var rules []NamespaceRule err = json.Unmarshal(rr.Body.Bytes(), &rules) require.NoError(t, err) require.Equal(t, http.StatusOK, rr.Code) require.Equal(t, contentTypeJSON, rr.Header().Get(hdrContentType)) require.Contains(t, rules, rule) } ================================================ FILE: server/http/common.go ================================================ package cayleyhttp import ( "encoding/json" "fmt" "net/http" "github.com/cayleygraph/cayley/graph" httpgraph "github.com/cayleygraph/cayley/graph/http" ) func jsonResponse(w http.ResponseWriter, code int, err interface{}) { w.Header().Set("Content-Type", contentTypeJSON) w.WriteHeader(code) w.Write([]byte(`{"error": `)) var s string switch err := err.(type) { case string: s = err case error: s = err.Error() default: s = fmt.Sprint(err) } data, _ := json.Marshal(s) w.Write(data) w.Write([]byte(`}`)) } // HandleForRequest returns new graph.Handle for given writer name, options and request func HandleForRequest(h *graph.Handle, wtyp string, wopt graph.Options, r *http.Request) (*graph.Handle, error) { g, ok := h.QuadStore.(httpgraph.QuadStore) if !ok { return h, nil } qs, err := g.ForRequest(r) if err != nil { return nil, err } qw, err := graph.NewQuadWriter(wtyp, qs, wopt) if err != nil { qs.Close() return nil, err } return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil } ================================================ FILE: ui/embed.go ================================================ package ui import "embed" //go:embed web var FS embed.FS ================================================ FILE: ui/web/asset-manifest.json ================================================ { "files": { "main.css": "/static/css/main.72fe53e6.chunk.css", "main.js": "/static/js/main.84d3ab8c.chunk.js", "main.js.map": "/static/js/main.84d3ab8c.chunk.js.map", "runtime-main.js": "/static/js/runtime-main.0686c6e7.js", "runtime-main.js.map": "/static/js/runtime-main.0686c6e7.js.map", "static/css/2.6b51f286.chunk.css": "/static/css/2.6b51f286.chunk.css", "static/js/2.7d84b3fa.chunk.js": "/static/js/2.7d84b3fa.chunk.js", "static/js/2.7d84b3fa.chunk.js.map": "/static/js/2.7d84b3fa.chunk.js.map", "index.html": "/index.html", "precache-manifest.5316e0e4a35813e95b82d4799f1bb55c.js": "/precache-manifest.5316e0e4a35813e95b82d4799f1bb55c.js", "service-worker.js": "/service-worker.js", "static/css/2.6b51f286.chunk.css.map": "/static/css/2.6b51f286.chunk.css.map", "static/css/main.72fe53e6.chunk.css.map": "/static/css/main.72fe53e6.chunk.css.map", "static/media/logo.svg": "/static/media/logo.ca711468.svg" }, "entrypoints": [ "static/js/runtime-main.0686c6e7.js", "static/css/2.6b51f286.chunk.css", "static/js/2.7d84b3fa.chunk.js", "static/css/main.72fe53e6.chunk.css", "static/js/main.84d3ab8c.chunk.js" ] } ================================================ FILE: ui/web/gizmo.d.ts ================================================ /** * Both `.Morphism()` and `.Vertex()` create path objects, which provide the following traversal methods. * Note that `.Vertex()` returns a query object, which is a subclass of path object. * * For these examples, suppose we have the following graph: * * ``` * +-------+ +------+ * | alice |----- ->| fred |<-- * +-------+ \---->+-------+-/ +------+ \-+-------+ * ----->| #bob# | | |*emily*| * +---------+--/ --->+-------+ | +-------+ * | charlie | / v * +---------+ / +--------+ * \--- +--------+ |*#greg#*| * \-->| #dani# |------------>+--------+ * +--------+ * ``` * * Where every link is a `` relationship, and the nodes with an extra `#` in the name have an extra `` link. As in, * * ``` * -- --> "cool_person" * ``` * * Perhaps these are the influencers in our community. So too are extra `*`s in the name -- these are our smart people, * according to the `` label, eg, the quad: * * ``` * "smart_person" . * ``` */ interface Path { /** Execute the query and adds the results, with all tags, as a string-to-string (tag to node) map in the output set, one for each path that a traversal could take. */ all(): void; /** Alias for intersect. */ and(path: Path): Path; /** Alias for tag. */ as(...tags: string[]): Path; /** Return current path to a set of nodes on a given tag, preserving all constraints. * If still valid, a path will now consider their vertex to be the same one as the previously tagged one, with the added constraint that it was valid all the way here. Useful for traversing back in queries and taking another route for things that have matched so far. */ back(tag?: string): Path; /** Follow the predicate in either direction. Same as out or in. */ both(path: Path, ...tags: string[]): Path; /** Return a number of results and returns it as a value. */ count(): number; /** Alias for Except */ difference(path: Path): Path; /** Removes all paths which match query from current path. In a set-theoretic sense, this is (A - B). While `g.V().Except(path)` to achieve `U - B = !B` is supported, it's often very slow. */ except(path: Path): Path; /** Apply constraints to a set of nodes. Can be used to filter values by range or match strings. */ filter(...args: any): Path; /** The way to use a path prepared with Morphism. Applies the path chain on the morphism object to the current path. * Starts as if at the g.M() and follows through the morphism path. */ follow(path: Path): Path; /** The same as follow but follows the chain in the reverse direction. Flips "In" and "Out" where appropriate, the net result being a virtual predicate followed in the reverse direction. Starts at the end of the morphism and follows it backwards (with appropriate flipped directions) to the g.M() location. */ followR(path: Path): Path; /** The same as follow but follows the chain recursively. Starts as if at the g.M() and follows through the morphism path multiple times, returning all nodes encountered. */ followRecursive(path: Path): Path; /** Call callback(data) for each result, where data is the tag-to-string map as in All case. * @param [limit] An integer value on the first `limit` paths to process. * @param callback: A javascript function of the form `function(data)` */ forEach(callback: (data: { [key: string]: any }) => void): void; forEach( limit: number, callback: (data: { [key: string]: any }) => void ): void; /** Alias for forEach. */ map(callback: (data: { [key: string]: any }) => void): void; map(limit: number, callback: (data: { [key: string]: any }) => void): void; /** The same as All, but limited to the first N unique nodes at the end of the path, and each of their possible traversals. */ getLimit(limit: number): void; /** Filter all paths which are, at this point, on the subject for the given predicate and object, but do not follow the path, merely filter the possible paths. Usually useful for starting with all nodes, or limiting to a subset depending on some predicate/value pair. * @param predicate A string for a predicate node. * @param object A string for a object node or a set of filters to find it. */ has(predicate: string, object: string): Path; /** The same as Has, but sets constraint in reverse direction. */ hasR(predicate: string, object: string): Path; /** The inverse of out. Starting with the nodes in `path` on the object, follow the quads with predicates defined by `predicatePath` to their subjects. * @param [predicatePath] One of: * * null or undefined: All predicates pointing into this node * * a string: The predicate name to follow into this node * * a list of strings: The predicates to follow into this node * * a query path object: The target of which is a set of predicates to follow. * @param [tags] One of: * * null or undefined: No tags * * a string: A single tag to add the predicate used to the output set. * * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. */ in(predicatePath?: Path, ...tags: string[]): Path; /** Get the list of predicates that are pointing in to a node. */ inPredicates(): Path; /** Filter all paths by the result of another query path. This is essentially a join where, at the stage of each path, a node is shared. */ intersect(path: Path): Path; /** Filter all paths to ones which, at this point, are on the given node. * @param node: A string for a node. Can be repeated or a list of strings. */ is(node: string, ...nodes: string[]): Path; /** Set (or remove) the subgraph context to consider in the following traversals. * Affects all in(), out(), and both() calls that follow it. The default LabelContext is null (all subgraphs). * @param predicatePath One of: * * null or undefined: In future traversals, consider all edges, regardless of subgraph. * * a string: The name of the subgraph to restrict traversals to. * * a list of strings: A set of subgraphs to restrict traversals to. * * a query path object: The target of which is a set of subgraphs. * @param tags One of: * * null or undefined: No tags * * a string: A single tag to add the last traversed label to the output set. * * a list of strings: Multiple tags to use as keys to save the label used to the output set. */ labelContext(labelPath: Path, ...tags: string[]): Path; /** Get the list of inbound and outbound quad labels */ labels(): Path; /** Limit a number of nodes for current path. */ limit(limit: number): Path; /** Alias for Union. */ or(path: Path): Path; /** The work-a-day way to get between nodes, in the forward direction. Starting with the nodes in `path` on the subject, follow the quads with predicates defined by `predicatePath` to their objects. * @param predicatePath (Optional): One of: * * null or undefined: All predicates pointing out from this node * * a string: The predicate name to follow out from this node * * a list of strings: The predicates to follow out from this node * * a query path object: The target of which is a set of predicates to follow. * @param tags (Optional): One of: * * null or undefined: No tags * * a string: A single tag to add the predicate used to the output set. * * a list of strings: Multiple tags to use as keys to save the predicate used to the output set. */ out(predicatePath?: Path, ...tags: string[]): Path; /** Get the list of predicates that are pointing out from a node. */ outPredicates(): Path; /** Save the object of all quads with predicate into tag, without traversal. * @param predicate A string for a predicate node. * @param tag A string for a tag key to store the object node. */ save(predicate: string, tag: string): Path; /** The same as save, but returns empty tags if predicate does not exists. */ saveOpt(predicate: string, tag: string): Path; /** The same as saveOpt, but tags values via reverse predicate. */ saveOptR(predicate: string, tag: string): Path; /** The same as save, but tags values via reverse predicate. */ saveR(predicate: string, tag: string): Path; /** Tag the list of predicates that are pointing in to a node. */ saveInPredicates(tag: string): Path; /** Tag the list of predicates that are pointing out from a node. */ saveOutPredicates(tag: string): Path; /** Skip a number of nodes for current path. * @param offset: A number of nodes to skip. */ skip(offset: number): Path; /** Save a list of nodes to a given tag. In order to save your work or learn more about how a path got to the end, we have tags. The simplest thing to do is to add a tag anywhere you'd like to put each node in the result set. * @param tag: A string or list of strings to act as a result key. The value for tag was the vertex the path was on at the time it reached "Tag" */ tag(...tags: string[]): Path; /** * The same as toArray, but instead of a list of top-level nodes, returns an Array of tag-to-string dictionaries, much as All would, except inside the JS environment. */ tagArray(): void; /** The same as TagArray, but limited to one result node. Returns a tag-to-string map. */ tagValue(): void; /** Execute a query and returns the results at the end of the query path as an JS array. */ toArray(): void; /** The same as ToArray, but limited to one result node. */ toValue(): void; /** Return the combined paths of the two queries. Notice that it's per-path, not per-node. Once again, if multiple paths reach the same destination, they might have had different ways of getting there (and different tags). See also: `Path.prototype.tag()` */ union(path: Path): Path; /** Remove duplicate values from the path. */ unique(): Path; } interface Graph { /** A shorthand for Vertex. */ V(...nodeId: string[]): Path; /** A shorthand for Morphism */ M(): Path; /** Start a query path at the given vertex/vertices. No ids means "all vertices". */ Vertex(...nodeId: string[]): Path; /** Create a morphism path object. Unqueryable on it's own, defines one end of the path. Saving these to variables with */ Morphism(): Path; /** Load all namespaces saved to graph. */ loadNamespaces(): void; /** Register all default namespaces for automatic IRI resolution. */ addDefaultNamespaces(): void; /** Associate prefix with a given IRI namespace. */ addNamespace(): void; /** Add data programmatically to the JSON result list. Can be any JSON type. */ emit(): void; /** Create an IRI values from a given string. */ IRI(): string; } /** This is the only special object in the environment, generates the query objects. Under the hood, they're simple objects that get compiled to a Go iterator tree when executed. */ declare var graph: Graph; /** Alias of graph. This is the only special object in the environment, generates the query objects. Under the hood, they're simple objects that get compiled to a Go iterator tree when executed. */ declare var g: Graph; interface RegexFilter {} /** Filter by match a regular expression ([syntax](https://github.com/google/re2/wiki/Syntax)). By default works only on literals unless includeIRIs is set to `true`. */ declare function regex(expression: string, includeIRIs?: boolean): RegexFilter; ================================================ FILE: ui/web/index.html ================================================ Cayley
================================================ FILE: ui/web/manifest.json ================================================ { "short_name": "Cayley", "name": "Cayley", "icons": [ { "src": "favicon.ico", "sizes": "64x64 32x32 24x24 16x16", "type": "image/x-icon" }, { "src": "logo192.png", "type": "image/png", "sizes": "192x192" }, { "src": "logo512.png", "type": "image/png", "sizes": "512x512" } ], "start_url": ".", "display": "standalone", "theme_color": "#000000", "background_color": "#ffffff" } ================================================ FILE: ui/web/precache-manifest.5316e0e4a35813e95b82d4799f1bb55c.js ================================================ self.__precacheManifest = (self.__precacheManifest || []).concat([ { "revision": "04b3493090e1efb6ffc5ded2f5343ae0", "url": "/index.html" }, { "revision": "8fcf0599cb1e0923f5dd", "url": "/static/css/2.6b51f286.chunk.css" }, { "revision": "c254b9a695b5dbe62b1f", "url": "/static/css/main.72fe53e6.chunk.css" }, { "revision": "8fcf0599cb1e0923f5dd", "url": "/static/js/2.7d84b3fa.chunk.js" }, { "revision": "c254b9a695b5dbe62b1f", "url": "/static/js/main.84d3ab8c.chunk.js" }, { "revision": "d18b22d979014e3125a8", "url": "/static/js/runtime-main.0686c6e7.js" }, { "revision": "ca71146870e23a16db96ef358169e8fa", "url": "/static/media/logo.ca711468.svg" } ]); ================================================ FILE: ui/web/robots.txt ================================================ # https://www.robotstxt.org/robotstxt.html User-agent: * ================================================ FILE: ui/web/service-worker.js ================================================ /** * Welcome to your Workbox-powered service worker! * * You'll need to register this file in your web app and you should * disable HTTP caching for this file too. * See https://goo.gl/nhQhGp * * The rest of the code is auto-generated. Please don't update this file * directly; instead, make changes to your Workbox build configuration * and re-run your build process. * See https://goo.gl/2aRDsh */ importScripts("https://storage.googleapis.com/workbox-cdn/releases/4.3.1/workbox-sw.js"); importScripts( "/precache-manifest.5316e0e4a35813e95b82d4799f1bb55c.js" ); self.addEventListener('message', (event) => { if (event.data && event.data.type === 'SKIP_WAITING') { self.skipWaiting(); } }); workbox.core.clientsClaim(); /** * The workboxSW.precacheAndRoute() method efficiently caches and responds to * requests for URLs in the manifest. * See https://goo.gl/S9QRab */ self.__precacheManifest = [].concat(self.__precacheManifest || []); workbox.precaching.precacheAndRoute(self.__precacheManifest, {}); workbox.routing.registerNavigationRoute(workbox.precaching.getCacheKeyForURL("/index.html"), { blacklist: [/^\/_/,/\/[^/?]+\.[^/]+$/], }); ================================================ FILE: ui/web/static/css/2.6b51f286.chunk.css ================================================ /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-card{border-radius:4px;background-color:#fff;background-color:var(--mdc-theme-surface,#fff);box-shadow:0 2px 1px -1px rgba(0,0,0,.2),0 1px 1px 0 rgba(0,0,0,.14),0 1px 3px 0 rgba(0,0,0,.12);display:flex;flex-direction:column;box-sizing:border-box}.mdc-card--outlined{box-shadow:0 0 0 0 rgba(0,0,0,.2),0 0 0 0 rgba(0,0,0,.14),0 0 0 0 rgba(0,0,0,.12);border:1px solid #e0e0e0}.mdc-card__media{position:relative;box-sizing:border-box;background-repeat:no-repeat;background-position:50%;background-size:cover}.mdc-card__media:before{display:block;content:""}.mdc-card__media:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.mdc-card__media:last-child{border-bottom-left-radius:inherit;border-bottom-right-radius:inherit}.mdc-card__media--square:before{margin-top:100%}.mdc-card__media--16-9:before{margin-top:56.25%}.mdc-card__media-content{position:absolute;top:0;right:0;bottom:0;left:0;box-sizing:border-box}.mdc-card__primary-action{display:flex;flex-direction:column;box-sizing:border-box;position:relative;outline:none;color:inherit;text-decoration:none;cursor:pointer;overflow:hidden}.mdc-card__primary-action:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.mdc-card__primary-action:last-child{border-bottom-left-radius:inherit;border-bottom-right-radius:inherit}.mdc-card__actions{display:flex;flex-direction:row;align-items:center;box-sizing:border-box;min-height:52px;padding:8px}.mdc-card__actions--full-bleed{padding:0}.mdc-card__action-buttons,.mdc-card__action-icons{display:flex;flex-direction:row;align-items:center;box-sizing:border-box}.mdc-card__action-icons{color:rgba(0,0,0,.6);flex-grow:1;justify-content:flex-end}.mdc-card__action-buttons+.mdc-card__action-icons{margin-left:16px;margin-right:0}.mdc-card__action-buttons+.mdc-card__action-icons[dir=rtl],[dir=rtl] .mdc-card__action-buttons+.mdc-card__action-icons{margin-left:0;margin-right:16px}.mdc-card__action{display:inline-flex;flex-direction:row;align-items:center;box-sizing:border-box;justify-content:center;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.mdc-card__action:focus{outline:none}.mdc-card__action--button{margin-left:0;margin-right:8px;padding:0 8px}.mdc-card__action--button[dir=rtl],[dir=rtl] .mdc-card__action--button{margin-left:8px;margin-right:0}.mdc-card__action--button:last-child,.mdc-card__action--button:last-child[dir=rtl],[dir=rtl] .mdc-card__action--button:last-child{margin-left:0;margin-right:0}.mdc-card__actions--full-bleed .mdc-card__action--button{justify-content:space-between;width:100%;height:auto;max-height:none;margin:0;padding:8px 16px;text-align:left}.mdc-card__actions--full-bleed .mdc-card__action--button[dir=rtl],[dir=rtl] .mdc-card__actions--full-bleed .mdc-card__action--button{text-align:right}.mdc-card__action--icon{margin:-6px 0;padding:12px}.mdc-card__action--icon:not(:disabled){color:rgba(0,0,0,.6)}.mdc-card__primary-action{--mdc-ripple-fg-size:0;--mdc-ripple-left:0;--mdc-ripple-top:0;--mdc-ripple-fg-scale:1;--mdc-ripple-fg-translate-end:0;--mdc-ripple-fg-translate-start:0;-webkit-tap-highlight-color:rgba(0,0,0,0);will-change:transform,opacity}.mdc-card__primary-action:after,.mdc-card__primary-action:before{position:absolute;border-radius:50%;opacity:0;pointer-events:none;content:""}.mdc-card__primary-action:before{-webkit-transition:opacity 15ms linear,background-color 15ms linear;transition:opacity 15ms linear,background-color 15ms linear;z-index:1}.mdc-card__primary-action.mdc-ripple-upgraded:before{-webkit-transform:scale(1);-webkit-transform:scale(var(--mdc-ripple-fg-scale,1));transform:scale(1);transform:scale(var(--mdc-ripple-fg-scale,1))}.mdc-card__primary-action.mdc-ripple-upgraded:after{top:0;left:0;-webkit-transform:scale(0);transform:scale(0);-webkit-transform-origin:center center;transform-origin:center center}.mdc-card__primary-action.mdc-ripple-upgraded--unbounded:after{top:0;top:var(--mdc-ripple-top,0);left:0;left:var(--mdc-ripple-left,0)}.mdc-card__primary-action.mdc-ripple-upgraded--foreground-activation:after{-webkit-animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards;animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards}.mdc-card__primary-action.mdc-ripple-upgraded--foreground-deactivation:after{-webkit-animation:mdc-ripple-fg-opacity-out .15s;animation:mdc-ripple-fg-opacity-out .15s;-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1));transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1))}.mdc-card__primary-action:after,.mdc-card__primary-action:before{top:-50%;left:-50%;width:200%;height:200%}.mdc-card__primary-action.mdc-ripple-upgraded:after{width:100%;width:var(--mdc-ripple-fg-size,100%);height:100%;height:var(--mdc-ripple-fg-size,100%)}.mdc-card__primary-action:after,.mdc-card__primary-action:before{background-color:#000}.mdc-card__primary-action:hover:before{opacity:.04}.mdc-card__primary-action.mdc-ripple-upgraded--background-focused:before,.mdc-card__primary-action:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-card__primary-action:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}.mdc-card__primary-action:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-card__primary-action.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.12} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-tab-scroller{overflow-y:hidden}.mdc-tab-scroller__test{position:absolute;top:-9999px;width:100px;height:100px;overflow-x:scroll}.mdc-tab-scroller__scroll-area{-webkit-overflow-scrolling:touch;display:flex;overflow-x:hidden}.mdc-tab-scroller__scroll-area::-webkit-scrollbar,.mdc-tab-scroller__test::-webkit-scrollbar{display:none}.mdc-tab-scroller__scroll-area--scroll{overflow-x:scroll}.mdc-tab-scroller__scroll-content{position:relative;display:flex;flex:1 0 auto;-webkit-transform:none;transform:none;will-change:transform}.mdc-tab-scroller--align-start .mdc-tab-scroller__scroll-content{justify-content:flex-start}.mdc-tab-scroller--align-end .mdc-tab-scroller__scroll-content{justify-content:flex-end}.mdc-tab-scroller--align-center .mdc-tab-scroller__scroll-content{justify-content:center}.mdc-tab-scroller--animating .mdc-tab-scroller__scroll-area{-webkit-overflow-scrolling:auto}.mdc-tab-scroller--animating .mdc-tab-scroller__scroll-content{transition:-webkit-transform .25s cubic-bezier(.4,0,.2,1);-webkit-transition:-webkit-transform .25s cubic-bezier(.4,0,.2,1);transition:transform .25s cubic-bezier(.4,0,.2,1);transition:transform .25s cubic-bezier(.4,0,.2,1),-webkit-transform .25s cubic-bezier(.4,0,.2,1)} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-tab-indicator{display:flex;position:absolute;top:0;left:0;width:100%;height:100%;pointer-events:none;z-index:1}.mdc-tab-indicator>.mdc-tab-indicator__content--underline{background-color:#6200ee;background-color:var(--mdc-theme-primary,#6200ee);height:2px}.mdc-tab-indicator>.mdc-tab-indicator__content--icon{color:#018786;color:var(--mdc-theme-secondary,#018786);height:34px;font-size:34px}.mdc-tab-indicator__content{-webkit-transform-origin:left;transform-origin:left;opacity:0}.mdc-tab-indicator__content--underline{align-self:flex-end;width:100%}.mdc-tab-indicator__content--icon{align-self:center;margin:0 auto}.mdc-tab-indicator--active>.mdc-tab-indicator__content{opacity:1}.mdc-tab-indicator>.mdc-tab-indicator__content{transition:-webkit-transform .25s cubic-bezier(.4,0,.2,1);-webkit-transition:-webkit-transform .25s cubic-bezier(.4,0,.2,1);transition:transform .25s cubic-bezier(.4,0,.2,1);transition:transform .25s cubic-bezier(.4,0,.2,1),-webkit-transform .25s cubic-bezier(.4,0,.2,1)}.mdc-tab-indicator--no-transition>.mdc-tab-indicator__content{-webkit-transition:none;transition:none}.mdc-tab-indicator--fade>.mdc-tab-indicator__content{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}.mdc-tab-indicator--active.mdc-tab-indicator--fade>.mdc-tab-indicator__content{-webkit-transition-delay:.1s;transition-delay:.1s} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-tab-bar{width:100%} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-tab{position:relative;font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:.875rem;line-height:2.25rem;font-weight:500;letter-spacing:.0892857143em;text-decoration:none;text-transform:uppercase;display:flex;flex:1 0 auto;justify-content:center;box-sizing:border-box;height:48px;padding:0 24px;border:none;outline:none;background:none;text-align:center;white-space:nowrap;cursor:pointer;-webkit-appearance:none;z-index:1}.mdc-tab .mdc-tab__icon,.mdc-tab .mdc-tab__text-label{color:#000;color:var(--mdc-theme-on-surface,#000)}.mdc-tab .mdc-tab__icon{fill:currentColor}.mdc-tab::-moz-focus-inner{padding:0;border:0}.mdc-tab--min-width{flex:0 1 auto}.mdc-tab__ripple{--mdc-ripple-fg-size:0;--mdc-ripple-left:0;--mdc-ripple-top:0;--mdc-ripple-fg-scale:1;--mdc-ripple-fg-translate-end:0;--mdc-ripple-fg-translate-start:0;-webkit-tap-highlight-color:rgba(0,0,0,0);will-change:transform,opacity;position:absolute;top:0;left:0;width:100%;height:100%;overflow:hidden}.mdc-tab__ripple:after,.mdc-tab__ripple:before{position:absolute;border-radius:50%;opacity:0;pointer-events:none;content:""}.mdc-tab__ripple:before{-webkit-transition:opacity 15ms linear,background-color 15ms linear;transition:opacity 15ms linear,background-color 15ms linear;z-index:1}.mdc-tab__ripple.mdc-ripple-upgraded:before{-webkit-transform:scale(1);-webkit-transform:scale(var(--mdc-ripple-fg-scale,1));transform:scale(1);transform:scale(var(--mdc-ripple-fg-scale,1))}.mdc-tab__ripple.mdc-ripple-upgraded:after{top:0;left:0;-webkit-transform:scale(0);transform:scale(0);-webkit-transform-origin:center center;transform-origin:center center}.mdc-tab__ripple.mdc-ripple-upgraded--unbounded:after{top:0;top:var(--mdc-ripple-top,0);left:0;left:var(--mdc-ripple-left,0)}.mdc-tab__ripple.mdc-ripple-upgraded--foreground-activation:after{-webkit-animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards;animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards}.mdc-tab__ripple.mdc-ripple-upgraded--foreground-deactivation:after{-webkit-animation:mdc-ripple-fg-opacity-out .15s;animation:mdc-ripple-fg-opacity-out .15s;-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1));transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1))}.mdc-tab__ripple:after,.mdc-tab__ripple:before{top:-50%;left:-50%;width:200%;height:200%}.mdc-tab__ripple.mdc-ripple-upgraded:after{width:100%;width:var(--mdc-ripple-fg-size,100%);height:100%;height:var(--mdc-ripple-fg-size,100%)}.mdc-tab__ripple:after,.mdc-tab__ripple:before{background-color:#6200ee}@supports not (-ms-ime-align:auto){.mdc-tab__ripple:after,.mdc-tab__ripple:before{background-color:#6200ee;background-color:var(--mdc-theme-primary,#6200ee)}}.mdc-tab__ripple:hover:before{opacity:.04}.mdc-tab__ripple.mdc-ripple-upgraded--background-focused:before,.mdc-tab__ripple:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-tab__ripple:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}.mdc-tab__ripple:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-tab__ripple.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.12}.mdc-tab__content{position:relative;display:flex;align-items:center;justify-content:center;height:inherit;pointer-events:none}.mdc-tab__icon,.mdc-tab__text-label{-webkit-transition:color .15s linear,opacity .15s linear;transition:color .15s linear,opacity .15s linear;z-index:2}.mdc-tab__text-label{display:inline-block;opacity:.6;line-height:1}.mdc-tab__icon{width:24px;height:24px;opacity:.54;font-size:24px}.mdc-tab--stacked{height:72px}.mdc-tab--stacked .mdc-tab__content{flex-direction:column;align-items:center;justify-content:space-between}.mdc-tab--stacked .mdc-tab__icon{padding-top:12px}.mdc-tab--stacked .mdc-tab__text-label{padding-bottom:16px}.mdc-tab--active .mdc-tab__icon,.mdc-tab--active .mdc-tab__text-label{color:#6200ee;color:var(--mdc-theme-primary,#6200ee)}.mdc-tab--active .mdc-tab__icon{fill:currentColor}.mdc-tab--active .mdc-tab__icon,.mdc-tab--active .mdc-tab__text-label{-webkit-transition-delay:.1s;transition-delay:.1s;opacity:1}.mdc-tab:not(.mdc-tab--stacked) .mdc-tab__icon+.mdc-tab__text-label{padding-left:8px;padding-right:0}.mdc-tab:not(.mdc-tab--stacked) .mdc-tab__icon+.mdc-tab__text-label[dir=rtl],[dir=rtl] .mdc-tab:not(.mdc-tab--stacked) .mdc-tab__icon+.mdc-tab__text-label{padding-left:0;padding-right:8px} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-typography{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased}.mdc-typography--headline1{font-size:6rem;line-height:6rem;letter-spacing:-.015625em}.mdc-typography--headline1,.mdc-typography--headline2{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-weight:300;text-decoration:inherit;text-transform:inherit}.mdc-typography--headline2{font-size:3.75rem;line-height:3.75rem;letter-spacing:-.0083333333em}.mdc-typography--headline3{font-size:3rem;line-height:3.125rem;letter-spacing:normal}.mdc-typography--headline3,.mdc-typography--headline4{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-weight:400;text-decoration:inherit;text-transform:inherit}.mdc-typography--headline4{font-size:2.125rem;line-height:2.5rem;letter-spacing:.0073529412em}.mdc-typography--headline5{font-size:1.5rem;font-weight:400;letter-spacing:normal}.mdc-typography--headline5,.mdc-typography--headline6{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;line-height:2rem;text-decoration:inherit;text-transform:inherit}.mdc-typography--headline6{font-size:1.25rem;font-weight:500;letter-spacing:.0125em}.mdc-typography--subtitle1{font-size:1rem;line-height:1.75rem;font-weight:400;letter-spacing:.009375em}.mdc-typography--subtitle1,.mdc-typography--subtitle2{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;text-decoration:inherit;text-transform:inherit}.mdc-typography--subtitle2{font-size:.875rem;line-height:1.375rem;font-weight:500;letter-spacing:.0071428571em}.mdc-typography--body1{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:1rem;line-height:1.5rem;font-weight:400;letter-spacing:.03125em;text-decoration:inherit;text-transform:inherit}.mdc-typography--body2{font-size:.875rem;letter-spacing:.0178571429em}.mdc-typography--body2,.mdc-typography--caption{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;line-height:1.25rem;font-weight:400;text-decoration:inherit;text-transform:inherit}.mdc-typography--caption{font-size:.75rem;letter-spacing:.0333333333em}.mdc-typography--button{font-size:.875rem;line-height:2.25rem;letter-spacing:.0892857143em}.mdc-typography--button,.mdc-typography--overline{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-weight:500;text-decoration:none;text-transform:uppercase}.mdc-typography--overline{font-size:.75rem;line-height:2rem;letter-spacing:.1666666667em} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */@-webkit-keyframes mdc-select-float-native-control{0%{-webkit-transform:translateY(8px);transform:translateY(8px);opacity:0}to{-webkit-transform:translateY(0);transform:translateY(0);opacity:1}}@keyframes mdc-select-float-native-control{0%{-webkit-transform:translateY(8px);transform:translateY(8px);opacity:0}to{-webkit-transform:translateY(0);transform:translateY(0);opacity:1}}.mdc-line-ripple{position:absolute;bottom:0;left:0;width:100%;height:2px;-webkit-transform:scaleX(0);transform:scaleX(0);transition:opacity .18s cubic-bezier(.4,0,.2,1),-webkit-transform .18s cubic-bezier(.4,0,.2,1);-webkit-transition:opacity .18s cubic-bezier(.4,0,.2,1),-webkit-transform .18s cubic-bezier(.4,0,.2,1);transition:transform .18s cubic-bezier(.4,0,.2,1),opacity .18s cubic-bezier(.4,0,.2,1);transition:transform .18s cubic-bezier(.4,0,.2,1),opacity .18s cubic-bezier(.4,0,.2,1),-webkit-transform .18s cubic-bezier(.4,0,.2,1);opacity:0;z-index:2}.mdc-line-ripple--active{-webkit-transform:scaleX(1);transform:scaleX(1);opacity:1}.mdc-line-ripple--deactivating{opacity:0}.mdc-notched-outline{display:flex;position:absolute;right:0;left:0;box-sizing:border-box;width:100%;max-width:100%;height:100%;text-align:left;pointer-events:none}.mdc-notched-outline[dir=rtl],[dir=rtl] .mdc-notched-outline{text-align:right}.mdc-notched-outline__leading,.mdc-notched-outline__notch,.mdc-notched-outline__trailing{box-sizing:border-box;height:100%;-webkit-transition:border .15s cubic-bezier(.4,0,.2,1);transition:border .15s cubic-bezier(.4,0,.2,1);border-top:1px solid;border-bottom:1px solid;pointer-events:none}.mdc-notched-outline__leading{border-left:1px solid;border-right:none;width:12px}.mdc-notched-outline__leading[dir=rtl],.mdc-notched-outline__trailing,[dir=rtl] .mdc-notched-outline__leading{border-left:none;border-right:1px solid}.mdc-notched-outline__trailing{flex-grow:1}.mdc-notched-outline__trailing[dir=rtl],[dir=rtl] .mdc-notched-outline__trailing{border-left:1px solid;border-right:none}.mdc-notched-outline__notch{flex:0 0 auto;width:auto;max-width:calc(100% - 24px)}.mdc-notched-outline .mdc-floating-label{display:inline-block;position:relative;top:17px;bottom:auto;max-width:100%}.mdc-notched-outline .mdc-floating-label--float-above{text-overflow:clip}.mdc-notched-outline--upgraded .mdc-floating-label--float-above{max-width:133.33333%}.mdc-notched-outline--notched .mdc-notched-outline__notch{padding-left:0;padding-right:8px;border-top:none}.mdc-notched-outline--notched .mdc-notched-outline__notch[dir=rtl],[dir=rtl] .mdc-notched-outline--notched .mdc-notched-outline__notch{padding-left:8px;padding-right:0}.mdc-notched-outline--no-label .mdc-notched-outline__notch{padding:0}.mdc-floating-label{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:1rem;line-height:1.75rem;font-weight:400;letter-spacing:.009375em;text-decoration:inherit;text-transform:inherit;position:absolute;left:0;-webkit-transform-origin:left top;transform-origin:left top;transition:color .15s cubic-bezier(.4,0,.2,1),-webkit-transform .15s cubic-bezier(.4,0,.2,1);-webkit-transition:color .15s cubic-bezier(.4,0,.2,1),-webkit-transform .15s cubic-bezier(.4,0,.2,1);transition:transform .15s cubic-bezier(.4,0,.2,1),color .15s cubic-bezier(.4,0,.2,1);transition:transform .15s cubic-bezier(.4,0,.2,1),color .15s cubic-bezier(.4,0,.2,1),-webkit-transform .15s cubic-bezier(.4,0,.2,1);line-height:1.15rem;text-align:left;text-overflow:ellipsis;white-space:nowrap;cursor:text;overflow:hidden;will-change:transform}.mdc-floating-label[dir=rtl],[dir=rtl] .mdc-floating-label{right:0;left:auto;-webkit-transform-origin:right top;transform-origin:right top;text-align:right}.mdc-floating-label--float-above{cursor:auto;-webkit-transform:translateY(-50%) scale(.75);transform:translateY(-50%) scale(.75)}.mdc-floating-label--shake{-webkit-animation:mdc-floating-label-shake-float-above-standard .25s 1;animation:mdc-floating-label-shake-float-above-standard .25s 1}@-webkit-keyframes mdc-floating-label-shake-float-above-standard{0%{-webkit-transform:translateX(0) translateY(-50%) scale(.75);transform:translateX(0) translateY(-50%) scale(.75)}33%{-webkit-animation-timing-function:cubic-bezier(.5,0,.701732,.495819);animation-timing-function:cubic-bezier(.5,0,.701732,.495819);-webkit-transform:translateX(4%) translateY(-50%) scale(.75);transform:translateX(4%) translateY(-50%) scale(.75)}66%{-webkit-animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);-webkit-transform:translateX(-4%) translateY(-50%) scale(.75);transform:translateX(-4%) translateY(-50%) scale(.75)}to{-webkit-transform:translateX(0) translateY(-50%) scale(.75);transform:translateX(0) translateY(-50%) scale(.75)}}@keyframes mdc-floating-label-shake-float-above-standard{0%{-webkit-transform:translateX(0) translateY(-50%) scale(.75);transform:translateX(0) translateY(-50%) scale(.75)}33%{-webkit-animation-timing-function:cubic-bezier(.5,0,.701732,.495819);animation-timing-function:cubic-bezier(.5,0,.701732,.495819);-webkit-transform:translateX(4%) translateY(-50%) scale(.75);transform:translateX(4%) translateY(-50%) scale(.75)}66%{-webkit-animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);-webkit-transform:translateX(-4%) translateY(-50%) scale(.75);transform:translateX(-4%) translateY(-50%) scale(.75)}to{-webkit-transform:translateX(0) translateY(-50%) scale(.75);transform:translateX(0) translateY(-50%) scale(.75)}}.mdc-select--with-leading-icon:not(.mdc-select--disabled) .mdc-select__icon{color:#000;color:var(--mdc-theme-on-surface,#000)}.mdc-select--with-leading-icon .mdc-select__icon{display:inline-block;position:absolute;bottom:16px;box-sizing:border-box;width:24px;height:24px;border:none;background-color:transparent;fill:currentColor;opacity:.54;text-decoration:none;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.mdc-select__icon:not([tabindex]),.mdc-select__icon[tabindex="-1"]{cursor:default;pointer-events:none}.mdc-select-helper-text{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:.75rem;line-height:1.25rem;font-weight:400;letter-spacing:.0333333333em;text-decoration:inherit;text-transform:inherit;display:block;line-height:normal;margin:0;-webkit-transition:opacity .18s cubic-bezier(.4,0,.2,1);transition:opacity .18s cubic-bezier(.4,0,.2,1);opacity:0;will-change:opacity}.mdc-select-helper-text:before{display:inline-block;width:0;height:16px;content:"";vertical-align:0}.mdc-select-helper-text--persistent{-webkit-transition:none;transition:none;opacity:1;will-change:auto}.mdc-select{--mdc-ripple-fg-size:0;--mdc-ripple-left:0;--mdc-ripple-top:0;--mdc-ripple-fg-scale:1;--mdc-ripple-fg-translate-end:0;--mdc-ripple-fg-translate-start:0;-webkit-tap-highlight-color:rgba(0,0,0,0);will-change:transform,opacity;display:inline-flex;position:relative;box-sizing:border-box;height:56px;overflow:hidden;will-change:opacity,transform,color}.mdc-select:not(.mdc-select--disabled){background-color:#f5f5f5}.mdc-select:after,.mdc-select:before{position:absolute;border-radius:50%;opacity:0;pointer-events:none;content:""}.mdc-select:before{-webkit-transition:opacity 15ms linear,background-color 15ms linear;transition:opacity 15ms linear,background-color 15ms linear;z-index:1}.mdc-select.mdc-ripple-upgraded:before{-webkit-transform:scale(1);-webkit-transform:scale(var(--mdc-ripple-fg-scale,1));transform:scale(1);transform:scale(var(--mdc-ripple-fg-scale,1))}.mdc-select.mdc-ripple-upgraded:after{top:0;left:0;-webkit-transform:scale(0);transform:scale(0);-webkit-transform-origin:center center;transform-origin:center center}.mdc-select.mdc-ripple-upgraded--unbounded:after{top:0;top:var(--mdc-ripple-top,0);left:0;left:var(--mdc-ripple-left,0)}.mdc-select.mdc-ripple-upgraded--foreground-activation:after{-webkit-animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards;animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards}.mdc-select.mdc-ripple-upgraded--foreground-deactivation:after{-webkit-animation:mdc-ripple-fg-opacity-out .15s;animation:mdc-ripple-fg-opacity-out .15s;-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1));transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1))}.mdc-select:after,.mdc-select:before{top:-50%;left:-50%;width:200%;height:200%}.mdc-select.mdc-ripple-upgraded:after{width:100%;width:var(--mdc-ripple-fg-size,100%);height:100%;height:var(--mdc-ripple-fg-size,100%)}.mdc-select:after,.mdc-select:before{background-color:rgba(0,0,0,.87)}.mdc-select:hover:before{opacity:.04}.mdc-select.mdc-ripple-upgraded--background-focused:before,.mdc-select:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-select:not(.mdc-select--disabled) .mdc-select__native-control,.mdc-select:not(.mdc-select--disabled) .mdc-select__selected-text{color:rgba(0,0,0,.87)}.mdc-select:not(.mdc-select--disabled) .mdc-floating-label{color:rgba(0,0,0,.6)}.mdc-select:not(.mdc-select--disabled) .mdc-select__native-control,.mdc-select:not(.mdc-select--disabled) .mdc-select__selected-text{border-bottom-color:rgba(0,0,0,.42)}.mdc-select:not(.mdc-select--disabled)+.mdc-select-helper-text{color:rgba(0,0,0,.6)}.mdc-select,.mdc-select__native-control{border-radius:4px 4px 0 0}.mdc-select:not(.mdc-select--disabled).mdc-select--focused .mdc-line-ripple{background-color:#6200ee;background-color:var(--mdc-theme-primary,#6200ee)}.mdc-select:not(.mdc-select--disabled).mdc-select--focused .mdc-floating-label{color:rgba(98,0,238,.87)}.mdc-select:not(.mdc-select--disabled) .mdc-select__native-control:hover{border-bottom-color:rgba(0,0,0,.87)}.mdc-select .mdc-floating-label--float-above{-webkit-transform:translateY(-70%) scale(.75);transform:translateY(-70%) scale(.75)}.mdc-select .mdc-floating-label{left:16px;right:auto;top:21px;pointer-events:none}.mdc-select .mdc-floating-label[dir=rtl],[dir=rtl] .mdc-select .mdc-floating-label{left:auto;right:16px}.mdc-select.mdc-select--with-leading-icon .mdc-floating-label{left:48px;right:auto}.mdc-select.mdc-select--with-leading-icon .mdc-floating-label[dir=rtl],[dir=rtl] .mdc-select.mdc-select--with-leading-icon .mdc-floating-label{left:auto;right:48px}.mdc-select.mdc-select--outlined .mdc-floating-label{left:4px;right:auto;top:17px}.mdc-select.mdc-select--outlined .mdc-floating-label[dir=rtl],[dir=rtl] .mdc-select.mdc-select--outlined .mdc-floating-label{left:auto;right:4px}.mdc-select.mdc-select--outlined.mdc-select--with-leading-icon .mdc-floating-label{left:36px;right:auto}.mdc-select.mdc-select--outlined.mdc-select--with-leading-icon .mdc-floating-label[dir=rtl],[dir=rtl] .mdc-select.mdc-select--outlined.mdc-select--with-leading-icon .mdc-floating-label{left:auto;right:36px}.mdc-select.mdc-select--outlined.mdc-select--with-leading-icon .mdc-floating-label--float-above{left:36px;right:auto}.mdc-select.mdc-select--outlined.mdc-select--with-leading-icon .mdc-floating-label--float-above[dir=rtl],[dir=rtl] .mdc-select.mdc-select--outlined.mdc-select--with-leading-icon .mdc-floating-label--float-above{left:auto;right:36px}.mdc-select__dropdown-icon{background:url("data:image/svg+xml;charset=utf-8,%3Csvg width='10' height='5' viewBox='7 10 10 5' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill-rule='evenodd' opacity='.54' d='M7 10l5 5 5-5z'/%3E%3C/svg%3E") no-repeat 50%;left:auto;right:8px;position:absolute;bottom:16px;width:24px;height:24px;transition:-webkit-transform .15s cubic-bezier(.4,0,.2,1);-webkit-transition:-webkit-transform .15s cubic-bezier(.4,0,.2,1);transition:transform .15s cubic-bezier(.4,0,.2,1);transition:transform .15s cubic-bezier(.4,0,.2,1),-webkit-transform .15s cubic-bezier(.4,0,.2,1);pointer-events:none}.mdc-select__dropdown-icon[dir=rtl],[dir=rtl] .mdc-select__dropdown-icon{left:8px;right:auto}.mdc-select--focused .mdc-select__dropdown-icon{background:url("data:image/svg+xml;charset=utf-8,%3Csvg width='10' height='5' viewBox='7 10 10 5' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='%236200ee' fill-rule='evenodd' d='M7 10l5 5 5-5z'/%3E%3C/svg%3E") no-repeat 50%;-webkit-transform:rotate(180deg) translateY(-5px);transform:rotate(180deg) translateY(-5px);transition:-webkit-transform .15s cubic-bezier(.4,0,.2,1);-webkit-transition:-webkit-transform .15s cubic-bezier(.4,0,.2,1);transition:transform .15s cubic-bezier(.4,0,.2,1);transition:transform .15s cubic-bezier(.4,0,.2,1),-webkit-transform .15s cubic-bezier(.4,0,.2,1)}.mdc-select__native-control{padding-top:20px}.mdc-select.mdc-select--focused .mdc-line-ripple:after{-webkit-transform:scaleY(2);transform:scaleY(2);opacity:1}.mdc-select+.mdc-select-helper-text{margin-right:12px;margin-left:12px}.mdc-select--outlined+.mdc-select-helper-text{margin-right:16px;margin-left:16px}.mdc-select--focused+.mdc-select-helper-text:not(.mdc-select-helper-text--validation-msg){opacity:1}.mdc-select__selected-text{min-width:200px;padding-top:22px}.mdc-select__native-control,.mdc-select__selected-text{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:1rem;line-height:1.75rem;font-weight:400;letter-spacing:.009375em;text-decoration:inherit;text-transform:inherit;box-sizing:border-box;width:100%;height:56px;padding:20px 52px 4px 16px;border:none;border-bottom:1px solid;outline:none;background-color:transparent;color:inherit;white-space:nowrap;cursor:pointer;-webkit-appearance:none;-moz-appearance:none;appearance:none}.mdc-select__native-control[dir=rtl],.mdc-select__selected-text[dir=rtl],[dir=rtl] .mdc-select__native-control,[dir=rtl] .mdc-select__selected-text{padding-left:52px;padding-right:16px}.mdc-select__native-control::-ms-expand,.mdc-select__selected-text::-ms-expand{display:none}.mdc-select__native-control::-ms-value,.mdc-select__selected-text::-ms-value{background-color:transparent;color:inherit}@-moz-document url-prefix(""){.mdc-select__native-control,.mdc-select__selected-text{text-indent:-2px}}.mdc-select--outlined{border:none;overflow:visible}.mdc-select--outlined:not(.mdc-select--disabled) .mdc-notched-outline__leading,.mdc-select--outlined:not(.mdc-select--disabled) .mdc-notched-outline__notch,.mdc-select--outlined:not(.mdc-select--disabled) .mdc-notched-outline__trailing{border-color:rgba(0,0,0,.24)}.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__native-control:hover~.mdc-notched-outline .mdc-notched-outline__leading,.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__native-control:hover~.mdc-notched-outline .mdc-notched-outline__notch,.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__native-control:hover~.mdc-notched-outline .mdc-notched-outline__trailing,.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__selected-text:hover~.mdc-notched-outline .mdc-notched-outline__leading,.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__selected-text:hover~.mdc-notched-outline .mdc-notched-outline__notch,.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__selected-text:hover~.mdc-notched-outline .mdc-notched-outline__trailing{border-color:rgba(0,0,0,.87)}.mdc-select--outlined:not(.mdc-select--disabled).mdc-select--focused .mdc-notched-outline .mdc-notched-outline__leading,.mdc-select--outlined:not(.mdc-select--disabled).mdc-select--focused .mdc-notched-outline .mdc-notched-outline__notch,.mdc-select--outlined:not(.mdc-select--disabled).mdc-select--focused .mdc-notched-outline .mdc-notched-outline__trailing{border-width:2px;border-color:#6200ee;border-color:var(--mdc-theme-primary,#6200ee)}.mdc-select--outlined .mdc-floating-label--shake{-webkit-animation:mdc-floating-label-shake-float-above-text-field-outlined .25s 1;animation:mdc-floating-label-shake-float-above-text-field-outlined .25s 1}.mdc-select--outlined .mdc-notched-outline .mdc-notched-outline__leading{border-radius:4px 0 0 4px}.mdc-select--outlined .mdc-notched-outline .mdc-notched-outline__leading[dir=rtl],.mdc-select--outlined .mdc-notched-outline .mdc-notched-outline__trailing,[dir=rtl] .mdc-select--outlined .mdc-notched-outline .mdc-notched-outline__leading{border-radius:0 4px 4px 0}.mdc-select--outlined .mdc-notched-outline .mdc-notched-outline__trailing[dir=rtl],[dir=rtl] .mdc-select--outlined .mdc-notched-outline .mdc-notched-outline__trailing{border-radius:4px 0 0 4px}.mdc-select--outlined .mdc-select__native-control{border-radius:4px}.mdc-select--outlined:after,.mdc-select--outlined:before{content:none}.mdc-select--outlined:not(.mdc-select--disabled){background-color:transparent}.mdc-select--outlined .mdc-floating-label--float-above{-webkit-transform:translateY(-144%) scale(1);transform:translateY(-144%) scale(1);font-size:.75rem}.mdc-select--outlined.mdc-notched-outline--upgraded .mdc-floating-label--float-above,.mdc-select--outlined .mdc-notched-outline--upgraded .mdc-floating-label--float-above{-webkit-transform:translateY(-130%) scale(.75);transform:translateY(-130%) scale(.75);font-size:1rem}.mdc-select--outlined .mdc-select__native-control,.mdc-select--outlined .mdc-select__selected-text{display:flex;padding:12px 52px 12px 16px;border:none;background-color:transparent;z-index:1}.mdc-select--outlined .mdc-select__native-control[dir=rtl],.mdc-select--outlined .mdc-select__selected-text[dir=rtl],[dir=rtl] .mdc-select--outlined .mdc-select__native-control,[dir=rtl] .mdc-select--outlined .mdc-select__selected-text{padding-left:52px;padding-right:16px}.mdc-select--outlined .mdc-select__selected-text{padding-top:14px}.mdc-select--outlined .mdc-select__icon{z-index:2}.mdc-select--outlined .mdc-floating-label{line-height:1.15rem;pointer-events:auto}.mdc-select--invalid:not(.mdc-select--disabled) .mdc-floating-label{color:#b00020;color:var(--mdc-theme-error,#b00020)}.mdc-select--invalid:not(.mdc-select--disabled) .mdc-select__native-control,.mdc-select--invalid:not(.mdc-select--disabled) .mdc-select__selected-text{border-bottom-color:#b00020;border-bottom-color:var(--mdc-theme-error,#b00020)}.mdc-select--invalid:not(.mdc-select--disabled).mdc-select--focused .mdc-line-ripple{background-color:#b00020;background-color:var(--mdc-theme-error,#b00020)}.mdc-select--invalid:not(.mdc-select--disabled).mdc-select--focused .mdc-floating-label{color:#b00020}.mdc-select--invalid:not(.mdc-select--disabled).mdc-select--invalid+.mdc-select-helper-text--validation-msg{color:#b00020;color:var(--mdc-theme-error,#b00020)}.mdc-select--invalid:not(.mdc-select--disabled) .mdc-select__native-control:hover{border-bottom-color:#b00020;border-bottom-color:var(--mdc-theme-error,#b00020)}.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled) .mdc-notched-outline__leading,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled) .mdc-notched-outline__notch,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled) .mdc-notched-outline__trailing,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__native-control:hover~.mdc-notched-outline .mdc-notched-outline__leading,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__native-control:hover~.mdc-notched-outline .mdc-notched-outline__notch,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__native-control:hover~.mdc-notched-outline .mdc-notched-outline__trailing,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__selected-text:hover~.mdc-notched-outline .mdc-notched-outline__leading,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__selected-text:hover~.mdc-notched-outline .mdc-notched-outline__notch,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled):not(.mdc-select--focused) .mdc-select__selected-text:hover~.mdc-notched-outline .mdc-notched-outline__trailing{border-color:#b00020;border-color:var(--mdc-theme-error,#b00020)}.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled).mdc-select--focused .mdc-notched-outline .mdc-notched-outline__leading,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled).mdc-select--focused .mdc-notched-outline .mdc-notched-outline__notch,.mdc-select--invalid.mdc-select--outlined:not(.mdc-select--disabled).mdc-select--focused .mdc-notched-outline .mdc-notched-outline__trailing{border-width:2px;border-color:#b00020;border-color:var(--mdc-theme-error,#b00020)}.mdc-select--invalid .mdc-select__dropdown-icon{background:url("data:image/svg+xml;charset=utf-8,%3Csvg width='10' height='5' viewBox='7 10 10 5' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='%23b00020' fill-rule='evenodd' d='M7 10l5 5 5-5z'/%3E%3C/svg%3E") no-repeat 50%}.mdc-select--invalid+.mdc-select-helper-text--validation-msg{opacity:1}.mdc-select--required .mdc-floating-label:after{content:"*"}.mdc-select--disabled{background-color:#fafafa;cursor:default;pointer-events:none}.mdc-select--disabled .mdc-floating-label{color:rgba(0,0,0,.37)}.mdc-select--disabled .mdc-select__dropdown-icon{background:url("data:image/svg+xml;charset=utf-8,%3Csvg width='10' height='5' viewBox='7 10 10 5' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill-rule='evenodd' opacity='.37' d='M7 10l5 5 5-5z'/%3E%3C/svg%3E") no-repeat 50%}.mdc-select--disabled .mdc-line-ripple{display:none}.mdc-select--disabled .mdc-select__icon{color:rgba(0,0,0,.37)}.mdc-select--disabled .mdc-select__native-control,.mdc-select--disabled .mdc-select__selected-text{color:rgba(0,0,0,.37);border-bottom-style:dotted}.mdc-select--disabled .mdc-select__selected-text{pointer-events:none}.mdc-select--disabled.mdc-select--outlined{background-color:transparent}.mdc-select--disabled.mdc-select--outlined .mdc-select__native-control,.mdc-select--disabled.mdc-select--outlined .mdc-select__selected-text{border-bottom-style:none}.mdc-select--disabled.mdc-select--outlined .mdc-notched-outline__leading,.mdc-select--disabled.mdc-select--outlined .mdc-notched-outline__notch,.mdc-select--disabled.mdc-select--outlined .mdc-notched-outline__trailing{border-color:rgba(0,0,0,.16)}.mdc-select--with-leading-icon .mdc-select__icon{left:16px;right:auto}.mdc-select--with-leading-icon .mdc-select__icon[dir=rtl],[dir=rtl] .mdc-select--with-leading-icon .mdc-select__icon{left:auto;right:16px}.mdc-select--with-leading-icon .mdc-select__native-control,.mdc-select--with-leading-icon .mdc-select__selected-text{padding-left:48px;padding-right:32px}.mdc-select--with-leading-icon .mdc-select__native-control[dir=rtl],.mdc-select--with-leading-icon .mdc-select__selected-text[dir=rtl],[dir=rtl] .mdc-select--with-leading-icon .mdc-select__native-control,[dir=rtl] .mdc-select--with-leading-icon .mdc-select__selected-text{padding-left:32px;padding-right:48px}.mdc-select--with-leading-icon.mdc-select--outlined .mdc-floating-label--float-above{-webkit-transform:translateY(-144%) translateX(-32px) scale(1);transform:translateY(-144%) translateX(-32px) scale(1)}.mdc-select--with-leading-icon.mdc-select--outlined .mdc-floating-label--float-above[dir=rtl],[dir=rtl] .mdc-select--with-leading-icon.mdc-select--outlined .mdc-floating-label--float-above{-webkit-transform:translateY(-144%) translateX(32px) scale(1);transform:translateY(-144%) translateX(32px) scale(1)}.mdc-select--with-leading-icon.mdc-select--outlined .mdc-floating-label--float-above{font-size:.75rem}.mdc-select--with-leading-icon.mdc-select--outlined.mdc-notched-outline--upgraded .mdc-floating-label--float-above,.mdc-select--with-leading-icon.mdc-select--outlined .mdc-notched-outline--upgraded .mdc-floating-label--float-above{-webkit-transform:translateY(-130%) translateX(-32px) scale(.75);transform:translateY(-130%) translateX(-32px) scale(.75)}.mdc-select--with-leading-icon.mdc-select--outlined.mdc-notched-outline--upgraded .mdc-floating-label--float-above[dir=rtl],.mdc-select--with-leading-icon.mdc-select--outlined .mdc-notched-outline--upgraded .mdc-floating-label--float-above[dir=rtl],[dir=rtl] .mdc-select--with-leading-icon.mdc-select--outlined.mdc-notched-outline--upgraded .mdc-floating-label--float-above,[dir=rtl] .mdc-select--with-leading-icon.mdc-select--outlined .mdc-notched-outline--upgraded .mdc-floating-label--float-above{-webkit-transform:translateY(-130%) translateX(32px) scale(.75);transform:translateY(-130%) translateX(32px) scale(.75)}.mdc-select--with-leading-icon.mdc-select--outlined.mdc-notched-outline--upgraded .mdc-floating-label--float-above,.mdc-select--with-leading-icon.mdc-select--outlined .mdc-notched-outline--upgraded .mdc-floating-label--float-above{font-size:1rem}.mdc-select--with-leading-icon.mdc-select--outlined .mdc-floating-label--shake{-webkit-animation:mdc-floating-label-shake-float-above-select-outlined-leading-icon .25s 1;animation:mdc-floating-label-shake-float-above-select-outlined-leading-icon .25s 1}.mdc-select--with-leading-icon.mdc-select--outlined[dir=rtl] .mdc-floating-label--shake,[dir=rtl] .mdc-select--with-leading-icon.mdc-select--outlined .mdc-floating-label--shake{-webkit-animation:mdc-floating-label-shake-float-above-select-outlined-leading-icon-rtl .25s 1;animation:mdc-floating-label-shake-float-above-select-outlined-leading-icon-rtl .25s 1}.mdc-select--with-leading-icon.mdc-select__menu .mdc-list-item__text,.mdc-select--with-leading-icon.mdc-select__menu .mdc-list-item__text[dir=rtl],[dir=rtl] .mdc-select--with-leading-icon.mdc-select__menu .mdc-list-item__text{padding-left:32px;padding-right:32px}.mdc-select__menu .mdc-list .mdc-list-item--selected{color:#000;color:var(--mdc-theme-on-surface,#000)}.mdc-select__menu .mdc-list .mdc-list-item--selected:after,.mdc-select__menu .mdc-list .mdc-list-item--selected:before{background-color:#000}@supports not (-ms-ime-align:auto){.mdc-select__menu .mdc-list .mdc-list-item--selected:after,.mdc-select__menu .mdc-list .mdc-list-item--selected:before{background-color:#000;background-color:var(--mdc-theme-on-surface,#000)}}.mdc-select__menu .mdc-list .mdc-list-item--selected:hover:before{opacity:.04}.mdc-select__menu .mdc-list .mdc-list-item--selected.mdc-ripple-upgraded--background-focused:before,.mdc-select__menu .mdc-list .mdc-list-item--selected:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-select__menu .mdc-list .mdc-list-item--selected:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}.mdc-select__menu .mdc-list .mdc-list-item--selected:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-select__menu .mdc-list .mdc-list-item--selected.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.12}@-webkit-keyframes mdc-floating-label-shake-float-above-select-outlined-leading-icon{0%{-webkit-transform:translateX(-32px) translateY(-130%) scale(.75);transform:translateX(-32px) translateY(-130%) scale(.75)}33%{-webkit-animation-timing-function:cubic-bezier(.5,0,.701732,.495819);animation-timing-function:cubic-bezier(.5,0,.701732,.495819);-webkit-transform:translateX(calc(4% - 32px)) translateY(-130%) scale(.75);transform:translateX(calc(4% - 32px)) translateY(-130%) scale(.75)}66%{-webkit-animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);-webkit-transform:translateX(calc(-4% - 32px)) translateY(-130%) scale(.75);transform:translateX(calc(-4% - 32px)) translateY(-130%) scale(.75)}to{-webkit-transform:translateX(-32px) translateY(-130%) scale(.75);transform:translateX(-32px) translateY(-130%) scale(.75)}}@keyframes mdc-floating-label-shake-float-above-select-outlined-leading-icon{0%{-webkit-transform:translateX(-32px) translateY(-130%) scale(.75);transform:translateX(-32px) translateY(-130%) scale(.75)}33%{-webkit-animation-timing-function:cubic-bezier(.5,0,.701732,.495819);animation-timing-function:cubic-bezier(.5,0,.701732,.495819);-webkit-transform:translateX(calc(4% - 32px)) translateY(-130%) scale(.75);transform:translateX(calc(4% - 32px)) translateY(-130%) scale(.75)}66%{-webkit-animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);-webkit-transform:translateX(calc(-4% - 32px)) translateY(-130%) scale(.75);transform:translateX(calc(-4% - 32px)) translateY(-130%) scale(.75)}to{-webkit-transform:translateX(-32px) translateY(-130%) scale(.75);transform:translateX(-32px) translateY(-130%) scale(.75)}}@-webkit-keyframes mdc-floating-label-shake-float-above-select-outlined-leading-icon-rtl{0%{-webkit-transform:translateX(32px) translateY(-130%) scale(.75);transform:translateX(32px) translateY(-130%) scale(.75)}33%{-webkit-animation-timing-function:cubic-bezier(.5,0,.701732,.495819);animation-timing-function:cubic-bezier(.5,0,.701732,.495819);-webkit-transform:translateX(calc(4% - -32px)) translateY(-130%) scale(.75);transform:translateX(calc(4% - -32px)) translateY(-130%) scale(.75)}66%{-webkit-animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);-webkit-transform:translateX(calc(-4% - -32px)) translateY(-130%) scale(.75);transform:translateX(calc(-4% - -32px)) translateY(-130%) scale(.75)}to{-webkit-transform:translateX(32px) translateY(-130%) scale(.75);transform:translateX(32px) translateY(-130%) scale(.75)}}@keyframes mdc-floating-label-shake-float-above-select-outlined-leading-icon-rtl{0%{-webkit-transform:translateX(32px) translateY(-130%) scale(.75);transform:translateX(32px) translateY(-130%) scale(.75)}33%{-webkit-animation-timing-function:cubic-bezier(.5,0,.701732,.495819);animation-timing-function:cubic-bezier(.5,0,.701732,.495819);-webkit-transform:translateX(calc(4% - -32px)) translateY(-130%) scale(.75);transform:translateX(calc(4% - -32px)) translateY(-130%) scale(.75)}66%{-webkit-animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);animation-timing-function:cubic-bezier(.302435,.381352,.55,.956352);-webkit-transform:translateX(calc(-4% - -32px)) translateY(-130%) scale(.75);transform:translateX(calc(-4% - -32px)) translateY(-130%) scale(.75)}to{-webkit-transform:translateX(32px) translateY(-130%) scale(.75);transform:translateX(32px) translateY(-130%) scale(.75)}} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-button{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:.875rem;line-height:2.25rem;font-weight:500;letter-spacing:.0892857143em;text-decoration:none;text-transform:uppercase;padding:0 8px;display:inline-flex;position:relative;align-items:center;justify-content:center;box-sizing:border-box;min-width:64px;height:36px;border:none;outline:none;line-height:inherit;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-appearance:none;overflow:hidden;vertical-align:middle;border-radius:4px}.mdc-button::-moz-focus-inner{padding:0;border:0}.mdc-button:active{outline:none}.mdc-button:hover{cursor:pointer}.mdc-button:disabled{background-color:transparent;color:rgba(0,0,0,.37);cursor:default;pointer-events:none}.mdc-button.mdc-button--dense{border-radius:4px}.mdc-button:not(:disabled){background-color:transparent}.mdc-button .mdc-button__icon{margin-left:0;margin-right:8px;display:inline-block;width:18px;height:18px;font-size:18px;vertical-align:top}.mdc-button .mdc-button__icon[dir=rtl],[dir=rtl] .mdc-button .mdc-button__icon{margin-left:8px;margin-right:0}.mdc-button:not(:disabled){color:#6200ee;color:var(--mdc-theme-primary,#6200ee)}.mdc-button__label+.mdc-button__icon{margin-left:8px;margin-right:0}.mdc-button__label+.mdc-button__icon[dir=rtl],[dir=rtl] .mdc-button__label+.mdc-button__icon{margin-left:0;margin-right:8px}svg.mdc-button__icon{fill:currentColor}.mdc-button--outlined .mdc-button__icon,.mdc-button--raised .mdc-button__icon,.mdc-button--unelevated .mdc-button__icon{margin-left:-4px;margin-right:8px}.mdc-button--outlined .mdc-button__icon[dir=rtl],.mdc-button--outlined .mdc-button__label+.mdc-button__icon,.mdc-button--raised .mdc-button__icon[dir=rtl],.mdc-button--raised .mdc-button__label+.mdc-button__icon,.mdc-button--unelevated .mdc-button__icon[dir=rtl],.mdc-button--unelevated .mdc-button__label+.mdc-button__icon,[dir=rtl] .mdc-button--outlined .mdc-button__icon,[dir=rtl] .mdc-button--raised .mdc-button__icon,[dir=rtl] .mdc-button--unelevated .mdc-button__icon{margin-left:8px;margin-right:-4px}.mdc-button--outlined .mdc-button__label+.mdc-button__icon[dir=rtl],.mdc-button--raised .mdc-button__label+.mdc-button__icon[dir=rtl],.mdc-button--unelevated .mdc-button__label+.mdc-button__icon[dir=rtl],[dir=rtl] .mdc-button--outlined .mdc-button__label+.mdc-button__icon,[dir=rtl] .mdc-button--raised .mdc-button__label+.mdc-button__icon,[dir=rtl] .mdc-button--unelevated .mdc-button__label+.mdc-button__icon{margin-left:-4px;margin-right:8px}.mdc-button--raised,.mdc-button--unelevated{padding:0 16px}.mdc-button--raised:disabled,.mdc-button--unelevated:disabled{background-color:rgba(0,0,0,.12);color:rgba(0,0,0,.37)}.mdc-button--raised:not(:disabled),.mdc-button--unelevated:not(:disabled){background-color:#6200ee}@supports not (-ms-ime-align:auto){.mdc-button--raised:not(:disabled),.mdc-button--unelevated:not(:disabled){background-color:#6200ee;background-color:var(--mdc-theme-primary,#6200ee)}}.mdc-button--raised:not(:disabled),.mdc-button--unelevated:not(:disabled){color:#fff;color:var(--mdc-theme-on-primary,#fff)}.mdc-button--raised{box-shadow:0 3px 1px -2px rgba(0,0,0,.2),0 2px 2px 0 rgba(0,0,0,.14),0 1px 5px 0 rgba(0,0,0,.12);-webkit-transition:box-shadow .28s cubic-bezier(.4,0,.2,1);transition:box-shadow .28s cubic-bezier(.4,0,.2,1)}.mdc-button--raised:focus,.mdc-button--raised:hover{box-shadow:0 2px 4px -1px rgba(0,0,0,.2),0 4px 5px 0 rgba(0,0,0,.14),0 1px 10px 0 rgba(0,0,0,.12)}.mdc-button--raised:active{box-shadow:0 5px 5px -3px rgba(0,0,0,.2),0 8px 10px 1px rgba(0,0,0,.14),0 3px 14px 2px rgba(0,0,0,.12)}.mdc-button--raised:disabled{box-shadow:0 0 0 0 rgba(0,0,0,.2),0 0 0 0 rgba(0,0,0,.14),0 0 0 0 rgba(0,0,0,.12)}.mdc-button--outlined{border-style:solid;padding:0 14px;border-width:2px}.mdc-button--outlined:disabled{border-color:rgba(0,0,0,.37)}.mdc-button--outlined:not(:disabled){border-color:#6200ee;border-color:var(--mdc-theme-primary,#6200ee)}.mdc-button--dense{height:32px;font-size:.8125rem}.mdc-button{--mdc-ripple-fg-size:0;--mdc-ripple-left:0;--mdc-ripple-top:0;--mdc-ripple-fg-scale:1;--mdc-ripple-fg-translate-end:0;--mdc-ripple-fg-translate-start:0;-webkit-tap-highlight-color:rgba(0,0,0,0);will-change:transform,opacity}.mdc-button:after,.mdc-button:before{position:absolute;border-radius:50%;opacity:0;pointer-events:none;content:""}.mdc-button:before{-webkit-transition:opacity 15ms linear,background-color 15ms linear;transition:opacity 15ms linear,background-color 15ms linear;z-index:1}.mdc-button.mdc-ripple-upgraded:before{-webkit-transform:scale(1);-webkit-transform:scale(var(--mdc-ripple-fg-scale,1));transform:scale(1);transform:scale(var(--mdc-ripple-fg-scale,1))}.mdc-button.mdc-ripple-upgraded:after{top:0;left:0;-webkit-transform:scale(0);transform:scale(0);-webkit-transform-origin:center center;transform-origin:center center}.mdc-button.mdc-ripple-upgraded--unbounded:after{top:0;top:var(--mdc-ripple-top,0);left:0;left:var(--mdc-ripple-left,0)}.mdc-button.mdc-ripple-upgraded--foreground-activation:after{-webkit-animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards;animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards}.mdc-button.mdc-ripple-upgraded--foreground-deactivation:after{-webkit-animation:mdc-ripple-fg-opacity-out .15s;animation:mdc-ripple-fg-opacity-out .15s;-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1));transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1))}.mdc-button:after,.mdc-button:before{top:-50%;left:-50%;width:200%;height:200%}.mdc-button.mdc-ripple-upgraded:after{width:100%;width:var(--mdc-ripple-fg-size,100%);height:100%;height:var(--mdc-ripple-fg-size,100%)}.mdc-button:after,.mdc-button:before{background-color:#6200ee}@supports not (-ms-ime-align:auto){.mdc-button:after,.mdc-button:before{background-color:#6200ee;background-color:var(--mdc-theme-primary,#6200ee)}}.mdc-button:hover:before{opacity:.04}.mdc-button.mdc-ripple-upgraded--background-focused:before,.mdc-button:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-button:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}.mdc-button:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}.mdc-button.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.12}.mdc-button--raised:after,.mdc-button--raised:before,.mdc-button--unelevated:after,.mdc-button--unelevated:before{background-color:#fff}@supports not (-ms-ime-align:auto){.mdc-button--raised:after,.mdc-button--raised:before,.mdc-button--unelevated:after,.mdc-button--unelevated:before{background-color:#fff;background-color:var(--mdc-theme-on-primary,#fff)}}.mdc-button--raised:hover:before,.mdc-button--unelevated:hover:before{opacity:.08}.mdc-button--raised.mdc-ripple-upgraded--background-focused:before,.mdc-button--raised:not(.mdc-ripple-upgraded):focus:before,.mdc-button--unelevated.mdc-ripple-upgraded--background-focused:before,.mdc-button--unelevated:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.24}.mdc-button--raised:not(.mdc-ripple-upgraded):after,.mdc-button--unelevated:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}.mdc-button--raised:not(.mdc-ripple-upgraded):active:after,.mdc-button--unelevated:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.24}.mdc-button--raised.mdc-ripple-upgraded,.mdc-button--unelevated.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.24} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-list{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:1rem;line-height:1.75rem;font-weight:400;letter-spacing:.009375em;text-decoration:inherit;text-transform:inherit;line-height:1.5rem;margin:0;padding:8px 0;list-style-type:none;color:rgba(0,0,0,.87);color:var(--mdc-theme-text-primary-on-background,rgba(0,0,0,.87))}.mdc-list-item__secondary-text{color:rgba(0,0,0,.54);color:var(--mdc-theme-text-secondary-on-background,rgba(0,0,0,.54))}.mdc-list-item__graphic{background-color:transparent;color:rgba(0,0,0,.38);color:var(--mdc-theme-text-icon-on-background,rgba(0,0,0,.38))}.mdc-list-item__meta{color:rgba(0,0,0,.38);color:var(--mdc-theme-text-hint-on-background,rgba(0,0,0,.38))}.mdc-list-group__subheader{color:rgba(0,0,0,.87);color:var(--mdc-theme-text-primary-on-background,rgba(0,0,0,.87))}.mdc-list--dense{padding-top:4px;padding-bottom:4px;font-size:.812rem}.mdc-list-item{display:flex;position:relative;align-items:center;justify-content:flex-start;height:48px;padding:0 16px;overflow:hidden}.mdc-list-item:focus{outline:none}.mdc-list-item--activated,.mdc-list-item--activated .mdc-list-item__graphic,.mdc-list-item--selected,.mdc-list-item--selected .mdc-list-item__graphic{color:#6200ee;color:var(--mdc-theme-primary,#6200ee)}.mdc-list-item--disabled{color:rgba(0,0,0,.38);color:var(--mdc-theme-text-disabled-on-background,rgba(0,0,0,.38))}.mdc-list-item__graphic{margin-left:0;margin-right:32px;width:24px;height:24px;flex-shrink:0;align-items:center;justify-content:center;fill:currentColor}.mdc-list-item[dir=rtl] .mdc-list-item__graphic,[dir=rtl] .mdc-list-item .mdc-list-item__graphic{margin-left:32px;margin-right:0}.mdc-list .mdc-list-item__graphic{display:inline-flex}.mdc-list-item__meta{margin-left:auto;margin-right:0}.mdc-list-item[dir=rtl] .mdc-list-item__meta,[dir=rtl] .mdc-list-item .mdc-list-item__meta{margin-left:0;margin-right:auto}.mdc-list-item__text{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.mdc-list-item__text[for]{pointer-events:none}.mdc-list-item__primary-text{text-overflow:ellipsis;white-space:nowrap;overflow:hidden;margin-top:0;line-height:normal;margin-bottom:-20px;display:block}.mdc-list-item__primary-text:before{display:inline-block;width:0;height:32px;content:"";vertical-align:0}.mdc-list-item__primary-text:after{display:inline-block;width:0;height:20px;content:"";vertical-align:-20px}.mdc-list--dense .mdc-list-item__primary-text{display:block;margin-top:0;line-height:normal;margin-bottom:-20px}.mdc-list--dense .mdc-list-item__primary-text:before{display:inline-block;width:0;height:24px;content:"";vertical-align:0}.mdc-list--dense .mdc-list-item__primary-text:after{display:inline-block;width:0;height:20px;content:"";vertical-align:-20px}.mdc-list-item__secondary-text{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:.875rem;line-height:1.25rem;font-weight:400;letter-spacing:.0178571429em;text-decoration:inherit;text-transform:inherit;text-overflow:ellipsis;white-space:nowrap;overflow:hidden;margin-top:0;line-height:normal;display:block}.mdc-list-item__secondary-text:before{display:inline-block;width:0;height:20px;content:"";vertical-align:0}.mdc-list--dense .mdc-list-item__secondary-text{display:block;margin-top:0;line-height:normal;font-size:inherit}.mdc-list--dense .mdc-list-item__secondary-text:before{display:inline-block;width:0;height:20px;content:"";vertical-align:0}.mdc-list--dense .mdc-list-item{height:40px}.mdc-list--dense .mdc-list-item__graphic{margin-left:0;margin-right:36px;width:20px;height:20px}.mdc-list-item[dir=rtl] .mdc-list--dense .mdc-list-item__graphic,[dir=rtl] .mdc-list-item .mdc-list--dense .mdc-list-item__graphic{margin-left:36px;margin-right:0}.mdc-list--avatar-list .mdc-list-item{height:56px}.mdc-list--avatar-list .mdc-list-item__graphic{margin-left:0;margin-right:16px;width:40px;height:40px;border-radius:50%}.mdc-list-item[dir=rtl] .mdc-list--avatar-list .mdc-list-item__graphic,[dir=rtl] .mdc-list-item .mdc-list--avatar-list .mdc-list-item__graphic{margin-left:16px;margin-right:0}.mdc-list--two-line .mdc-list-item__text{align-self:flex-start}.mdc-list--two-line .mdc-list-item{height:72px}.mdc-list--avatar-list.mdc-list--dense .mdc-list-item,.mdc-list--two-line.mdc-list--dense .mdc-list-item{height:60px}.mdc-list--avatar-list.mdc-list--dense .mdc-list-item__graphic{margin-left:0;margin-right:20px;width:36px;height:36px}.mdc-list-item[dir=rtl] .mdc-list--avatar-list.mdc-list--dense .mdc-list-item__graphic,[dir=rtl] .mdc-list-item .mdc-list--avatar-list.mdc-list--dense .mdc-list-item__graphic{margin-left:20px;margin-right:0}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item{cursor:pointer}a.mdc-list-item{color:inherit;text-decoration:none}.mdc-list-divider{height:0;margin:0;border:none;border-bottom:1px solid;border-bottom-color:rgba(0,0,0,.12)}.mdc-list-divider--padded{margin:0 16px}.mdc-list-divider--inset{margin-left:72px;margin-right:0;width:calc(100% - 72px)}.mdc-list-group[dir=rtl] .mdc-list-divider--inset,[dir=rtl] .mdc-list-group .mdc-list-divider--inset{margin-left:0;margin-right:72px}.mdc-list-divider--inset.mdc-list-divider--padded{width:calc(100% - 88px)}.mdc-list-group .mdc-list{padding:0}.mdc-list-group__subheader{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:1rem;line-height:1.75rem;font-weight:400;letter-spacing:.009375em;text-decoration:inherit;text-transform:inherit;margin:.75rem 16px}@-webkit-keyframes mdc-ripple-fg-radius-in{0%{-webkit-animation-timing-function:cubic-bezier(.4,0,.2,1);animation-timing-function:cubic-bezier(.4,0,.2,1);-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-start,0)) scale(1);transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-start,0)) scale(1)}to{-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1));transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1))}}@keyframes mdc-ripple-fg-radius-in{0%{-webkit-animation-timing-function:cubic-bezier(.4,0,.2,1);animation-timing-function:cubic-bezier(.4,0,.2,1);-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-start,0)) scale(1);transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-start,0)) scale(1)}to{-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1));transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1))}}@-webkit-keyframes mdc-ripple-fg-opacity-in{0%{-webkit-animation-timing-function:linear;animation-timing-function:linear;opacity:0}to{opacity:0;opacity:var(--mdc-ripple-fg-opacity,0)}}@keyframes mdc-ripple-fg-opacity-in{0%{-webkit-animation-timing-function:linear;animation-timing-function:linear;opacity:0}to{opacity:0;opacity:var(--mdc-ripple-fg-opacity,0)}}@-webkit-keyframes mdc-ripple-fg-opacity-out{0%{-webkit-animation-timing-function:linear;animation-timing-function:linear;opacity:0;opacity:var(--mdc-ripple-fg-opacity,0)}to{opacity:0}}@keyframes mdc-ripple-fg-opacity-out{0%{-webkit-animation-timing-function:linear;animation-timing-function:linear;opacity:0;opacity:var(--mdc-ripple-fg-opacity,0)}to{opacity:0}}.mdc-ripple-surface--test-edge-var-bug{--mdc-ripple-surface-test-edge-var:1px solid #000;visibility:hidden}.mdc-ripple-surface--test-edge-var-bug:before{border:var(--mdc-ripple-surface-test-edge-var)}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item{--mdc-ripple-fg-size:0;--mdc-ripple-left:0;--mdc-ripple-top:0;--mdc-ripple-fg-scale:1;--mdc-ripple-fg-translate-end:0;--mdc-ripple-fg-translate-start:0;-webkit-tap-highlight-color:rgba(0,0,0,0);will-change:transform,opacity}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:after,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:before{position:absolute;border-radius:50%;opacity:0;pointer-events:none;content:""}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:before{-webkit-transition:opacity 15ms linear,background-color 15ms linear;transition:opacity 15ms linear,background-color 15ms linear;z-index:1}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded:before{-webkit-transform:scale(1);-webkit-transform:scale(var(--mdc-ripple-fg-scale,1));transform:scale(1);transform:scale(var(--mdc-ripple-fg-scale,1))}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded:after{top:0;left:0;-webkit-transform:scale(0);transform:scale(0);-webkit-transform-origin:center center;transform-origin:center center}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded--unbounded:after{top:0;top:var(--mdc-ripple-top,0);left:0;left:var(--mdc-ripple-left,0)}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded--foreground-activation:after{-webkit-animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards;animation:mdc-ripple-fg-radius-in 225ms forwards,mdc-ripple-fg-opacity-in 75ms forwards}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded--foreground-deactivation:after{-webkit-animation:mdc-ripple-fg-opacity-out .15s;animation:mdc-ripple-fg-opacity-out .15s;-webkit-transform:translate(0) scale(1);-webkit-transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1));transform:translate(0) scale(1);transform:translate(var(--mdc-ripple-fg-translate-end,0)) scale(var(--mdc-ripple-fg-scale,1))}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:after,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:before{top:-50%;left:-50%;width:200%;height:200%}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded:after{width:100%;width:var(--mdc-ripple-fg-size,100%);height:100%;height:var(--mdc-ripple-fg-size,100%)}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:after,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:before{background-color:#000}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:hover:before{opacity:.04}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded--background-focused:before,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.12}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.12}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:before{opacity:.12}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:after,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:before{background-color:#6200ee}@supports not (-ms-ime-align:auto){:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:after,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:before{background-color:#6200ee;background-color:var(--mdc-theme-primary,#6200ee)}}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:hover:before{opacity:.16}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated.mdc-ripple-upgraded--background-focused:before,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.24}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.24}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--activated.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.24}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:before{opacity:.08}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:after,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:before{background-color:#6200ee}@supports not (-ms-ime-align:auto){:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:after,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:before{background-color:#6200ee;background-color:var(--mdc-theme-primary,#6200ee)}}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:hover:before{opacity:.12}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected.mdc-ripple-upgraded--background-focused:before,:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:not(.mdc-ripple-upgraded):focus:before{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.2}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:not(.mdc-ripple-upgraded):after{-webkit-transition:opacity .15s linear;transition:opacity .15s linear}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected:not(.mdc-ripple-upgraded):active:after{-webkit-transition-duration:75ms;transition-duration:75ms;opacity:.2}:not(.mdc-list--non-interactive)>:not(.mdc-list-item--disabled).mdc-list-item--selected.mdc-ripple-upgraded{--mdc-ripple-fg-opacity:0.2}.rmwc-icon--image{min-width:1em;min-height:1em;background-repeat:no-repeat;font-size:1.5rem;background-size:1em;background-position:50%}.rmwc-icon--size-xsmall{font-size:1.125rem;width:1em;height:1em}.rmwc-icon--size-small{font-size:1.25rem;width:1em;height:1em}.rmwc-icon--size-medium{font-size:1.5rem;width:1em;height:1em}.rmwc-icon--size-large{font-size:2.25rem;width:1em;height:1em}.rmwc-icon--size-xlarge{font-size:3rem;width:1em;height:1em} /*! Material Components for the Web Copyright (c) 2019 Google Inc. License: MIT */.mdc-drawer{background-color:#fff;border-radius:0 0 0 0;z-index:6;width:256px;display:flex;flex-direction:column;flex-shrink:0;box-sizing:border-box;height:100%;transition-property:-webkit-transform;-webkit-transition-property:-webkit-transform;transition-property:transform;transition-property:transform,-webkit-transform;-webkit-transition-timing-function:cubic-bezier(.4,0,.2,1);transition-timing-function:cubic-bezier(.4,0,.2,1);border-right:1px solid;border-color:rgba(0,0,0,.12);overflow:hidden}.mdc-drawer .mdc-drawer__title{color:rgba(0,0,0,.87)}.mdc-drawer .mdc-drawer__subtitle,.mdc-drawer .mdc-list-group__subheader,.mdc-drawer .mdc-list-item__graphic{color:rgba(0,0,0,.6)}.mdc-drawer .mdc-list-item{color:rgba(0,0,0,.87)}.mdc-drawer .mdc-list-item--activated .mdc-list-item__graphic{color:#6200ee}.mdc-drawer .mdc-list-item--activated{color:rgba(98,0,238,.87)}.mdc-drawer[dir=rtl],[dir=rtl] .mdc-drawer{border-radius:0 0 0 0}.mdc-drawer .mdc-list-item{border-radius:4px}.mdc-drawer.mdc-drawer--open:not(.mdc-drawer--closing)+.mdc-drawer-app-content{margin-left:256px;margin-right:0}.mdc-drawer.mdc-drawer--open:not(.mdc-drawer--closing)+.mdc-drawer-app-content[dir=rtl],[dir=rtl] .mdc-drawer.mdc-drawer--open:not(.mdc-drawer--closing)+.mdc-drawer-app-content{margin-left:0;margin-right:256px}.mdc-drawer[dir=rtl],[dir=rtl] .mdc-drawer{border-right-width:0;border-left-width:1px;border-right-style:none;border-left-style:solid}.mdc-drawer .mdc-list-item{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:.875rem;line-height:1.375rem;font-weight:500;letter-spacing:.0071428571em;text-decoration:inherit;text-transform:inherit;height:40px;margin:8px;padding:0 8px}.mdc-drawer .mdc-list-item:first-child{margin-top:2px}.mdc-drawer .mdc-list-item:last-child{margin-bottom:0}.mdc-drawer .mdc-list-group__subheader{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:.875rem;line-height:1.25rem;font-weight:400;letter-spacing:.0178571429em;text-decoration:inherit;text-transform:inherit;display:block;line-height:normal;margin:0;padding:0 16px}.mdc-drawer .mdc-list-group__subheader:before{display:inline-block;width:0;height:24px;content:"";vertical-align:0}.mdc-drawer .mdc-list-divider{margin:3px 0 4px}.mdc-drawer .mdc-list-item__graphic,.mdc-drawer .mdc-list-item__text{pointer-events:none}.mdc-drawer--animate{-webkit-transform:translateX(-100%);transform:translateX(-100%)}.mdc-drawer--animate[dir=rtl],[dir=rtl] .mdc-drawer--animate{-webkit-transform:translateX(100%);transform:translateX(100%)}.mdc-drawer--opening{-webkit-transition-duration:.25s;transition-duration:.25s}.mdc-drawer--opening,.mdc-drawer--opening[dir=rtl],[dir=rtl] .mdc-drawer--opening{-webkit-transform:translateX(0);transform:translateX(0)}.mdc-drawer--closing{-webkit-transform:translateX(-100%);transform:translateX(-100%);-webkit-transition-duration:.2s;transition-duration:.2s}.mdc-drawer--closing[dir=rtl],[dir=rtl] .mdc-drawer--closing{-webkit-transform:translateX(100%);transform:translateX(100%)}.mdc-drawer__header{flex-shrink:0;box-sizing:border-box;min-height:64px;padding:0 16px 4px}.mdc-drawer__title{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:1.25rem;line-height:2rem;font-weight:500;letter-spacing:.0125em;text-decoration:inherit;text-transform:inherit;display:block;margin-top:0;line-height:normal;margin-bottom:-20px}.mdc-drawer__title:before{display:inline-block;width:0;height:36px;content:"";vertical-align:0}.mdc-drawer__title:after{display:inline-block;width:0;height:20px;content:"";vertical-align:-20px}.mdc-drawer__subtitle{font-family:Roboto,sans-serif;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;font-size:.875rem;line-height:1.25rem;font-weight:400;letter-spacing:.0178571429em;text-decoration:inherit;text-transform:inherit;display:block;margin-top:0;line-height:normal;margin-bottom:0}.mdc-drawer__subtitle:before{display:inline-block;width:0;height:20px;content:"";vertical-align:0}.mdc-drawer__content{height:100%;overflow-y:auto;-webkit-overflow-scrolling:touch}.mdc-drawer--dismissible{left:0;right:auto;display:none;position:absolute}.mdc-drawer--dismissible[dir=rtl],[dir=rtl] .mdc-drawer--dismissible{left:auto;right:0}.mdc-drawer--dismissible.mdc-drawer--open{display:flex}.mdc-drawer-app-content{position:relative}.mdc-drawer-app-content,.mdc-drawer-app-content[dir=rtl],[dir=rtl] .mdc-drawer-app-content{margin-left:0;margin-right:0}.mdc-drawer--modal{box-shadow:0 8px 10px -5px rgba(0,0,0,.2),0 16px 24px 2px rgba(0,0,0,.14),0 6px 30px 5px rgba(0,0,0,.12);left:0;right:auto;display:none;position:fixed}.mdc-drawer--modal+.mdc-drawer-scrim{background-color:rgba(0,0,0,.32)}.mdc-drawer--modal[dir=rtl],[dir=rtl] .mdc-drawer--modal{left:auto;right:0}.mdc-drawer--modal.mdc-drawer--open{display:flex}.mdc-drawer-scrim{display:none;position:fixed;top:0;left:0;width:100%;height:100%;-webkit-transition-property:opacity;transition-property:opacity;-webkit-transition-timing-function:cubic-bezier(.4,0,.2,1);transition-timing-function:cubic-bezier(.4,0,.2,1);z-index:5}.mdc-drawer--open+.mdc-drawer-scrim{display:block}.mdc-drawer--animate+.mdc-drawer-scrim{opacity:0}.mdc-drawer--opening+.mdc-drawer-scrim{-webkit-transition-duration:.25s;transition-duration:.25s;opacity:1}.mdc-drawer--closing+.mdc-drawer-scrim{-webkit-transition-duration:.2s;transition-duration:.2s;opacity:0} /*# sourceMappingURL=2.6b51f286.chunk.css.map */ ================================================ FILE: ui/web/static/css/main.72fe53e6.chunk.css ================================================ @import url(https://fonts.googleapis.com/icon?family=Material+Icons);body{margin:0;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}.QueryHistory{height:300px;overflow:scroll}.QueryHistory .mdc-list-item{display:flex}.QueryHistory .query{height:2em;font-family:monospace;white-space:nowrap;flex:1 1}.QueryHistory .time{flex-shrink:0}.QueryHistory .language,.QueryHistory .status{margin:0 16px}main{display:flex;flex-direction:column}.graph{width:100%;flex:1 1}.graph svg{height:100%;width:100%}.App{height:100vh;width:100vw;background:#eee;box-sizing:border-box;overflow:hidden;display:flex;flex-direction:row}.App .Logo{height:2rem;margin-right:8px;position:relative;top:.5rem;display:inline-block}.App>main{flex:1 1;overflow:hidden;height:100%;box-sizing:border-box}.App>main .mdc-typography--headline6{margin:0 30px;padding:12px 0;display:block}.App>main .actions{background:#fff;padding:8px 24px;border-color:#d3d3d3;border-style:solid;border-width:1px 0}.App main .actions>.mdc-button{margin-right:24px}.App main .actions>.mdc-button:not(:first-child){margin-left:24px} /*# sourceMappingURL=main.72fe53e6.chunk.css.map */ ================================================ FILE: ui/web/static/js/2.7d84b3fa.chunk.js ================================================ (this["webpackJsonpcayley-ui"]=this["webpackJsonpcayley-ui"]||[]).push([[2],[function(e,t,n){"use strict";e.exports=n(48)},function(e,t,n){"use strict";n.d(t,"b",function(){return o}),n.d(t,"a",function(){return i}),n.d(t,"d",function(){return a}),n.d(t,"c",function(){return l});var r=function(e,t){return(r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(e,t)};function o(e,t){function n(){this.constructor=e}r(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)}var i=function(){return(i=Object.assign||function(e){for(var t,n=1,r=arguments.length;n=e.length&&(e=void 0),{value:e&&e[n++],done:!e}}}}function l(e,t){var n="function"===typeof Symbol&&e[Symbol.iterator];if(!n)return e;var r,o,i=n.call(e),a=[];try{for(;(void 0===t||t-- >0)&&!(r=i.next()).done;)a.push(r.value)}catch(l){o={error:l}}finally{try{r&&!r.done&&(n=i.return)&&n.call(i)}finally{if(o)throw o.error}}return a}},function(e,t,n){"use strict";n.d(t,"b",function(){return l});var r=n(37),o=n.n(r),i=n(8),a=n.n(i);n.d(t,"a",function(){return a.a});var l=o.a},function(e,t,n){e.exports=n(61)()},function(e,t,n){"use strict";function r(e,t){return function(e){if(Array.isArray(e))return e}(e)||function(e,t){if(Symbol.iterator in Object(e)||"[object Arguments]"===Object.prototype.toString.call(e)){var n=[],r=!0,o=!1,i=void 0;try{for(var a,l=e[Symbol.iterator]();!(r=(a=l.next()).done)&&(n.push(a.value),!t||n.length!==t);r=!0);}catch(u){o=!0,i=u}finally{try{r||null==l.return||l.return()}finally{if(o)throw i}}return n}}(e,t)||function(){throw new TypeError("Invalid attempt to destructure non-iterable instance")}()}n.d(t,"a",function(){return r})},function(e,t,n){"use strict";n.d(t,"a",function(){return p});var r=n(0),o=n(23),i=n(2),a=n(6),l=n(13),u=function(){return(u=Object.assign||function(e){for(var t,n=1,r=arguments.length;n"},o);Object(l.a)("Icon component prop 'iconOptions' is deprecated. You options should now be passed directly to the 'icon' prop. I.E. icon={"+JSON.stringify(p)+"}")}var h=u({},function(e){return r.isValidElement(e)||e&&"object"!==typeof e?{icon:e}:e}(n),o),m=h.icon,y=h.strategy,v=h.prefix,g=h.basename,b=h.render,_=h.size,E=s(h,["icon","strategy","prefix","basename","render","size"]),T=a.icon||{},C=T.basename,O=void 0===C?null:C,w=T.prefix,S=void 0===w?null:w,x=T.strategy,I=void 0===x?null:x,A=T.render,k=void 0===A?null:A,P=m,N=c(P,y||null,I||null),L=void 0===g?O:g,R="className"===N&&"string"===typeof m?""+String(v||S)+m:null,D="custom"===N?b||k:!!N&&f[N]||null;if(!D)return console.error("Icon: rendering not implemented for "+String(N)+"."),null;var M=D(u({},d,E,{content:P,className:Object(i.a)("rmwc-icon",L,d.className,E.className,R,(t={},t["rmwc-icon--size-"+(_||"")]=!!_,t))}));return M.props.children&&M.props.children.type&&["Avatar","Icon"].includes(M.props.children.type.displayName)?r.cloneElement(M.props.children,u({},M.props.children.props,M.props,{children:M.props.children.props.children,className:Object(i.a)(M.props.className,M.props.children.props.className)})):M});p.displayName="Icon"},function(e,t,n){"use strict";n.d(t,"a",function(){return d});var r=n(0),o=n(8),i=n.n(o),a=n(19),l=n(13),u=function(){return(u=Object.assign||function(e){for(var t,n=1,r=arguments.length;n0)&&!(r=i.next()).done;)a.push(r.value)}catch(l){o={error:l}}finally{try{r&&!r.done&&(n=i.return)&&n.call(i)}finally{if(o)throw o.error}}return a},f=function(){for(var e=[],t=0;t0&&v.some(function(e){return t.adapter_.containsEventTarget(e)})?this.resetActivationState_():(void 0!==e&&(v.push(e.target),this.registerDeactivationHandlers_(e)),n.wasElementMadeActive=this.checkElementMadeActive_(e),n.wasElementMadeActive&&this.animateActivation_(),requestAnimationFrame(function(){v=[],n.wasElementMadeActive||void 0===e||" "!==e.key&&32!==e.keyCode||(n.wasElementMadeActive=t.checkElementMadeActive_(e),n.wasElementMadeActive&&t.animateActivation_()),n.wasElementMadeActive||(t.activationState_=t.defaultActivationState_())}))}}},t.prototype.checkElementMadeActive_=function(e){return void 0===e||"keydown"!==e.type||this.adapter_.isSurfaceActive()},t.prototype.animateActivation_=function(){var e=this,n=t.strings,r=n.VAR_FG_TRANSLATE_START,o=n.VAR_FG_TRANSLATE_END,i=t.cssClasses,a=i.FG_DEACTIVATION,l=i.FG_ACTIVATION,u=t.numbers.DEACTIVATION_TIMEOUT_MS;this.layoutInternal_();var s="",c="";if(!this.adapter_.isUnbounded()){var f=this.getFgTranslationCoordinates_(),d=f.startPoint,p=f.endPoint;s=d.x+"px, "+d.y+"px",c=p.x+"px, "+p.y+"px"}this.adapter_.updateCssVariable(r,s),this.adapter_.updateCssVariable(o,c),clearTimeout(this.activationTimer_),clearTimeout(this.fgDeactivationRemovalTimer_),this.rmBoundedActivationClasses_(),this.adapter_.removeClass(a),this.adapter_.computeBoundingRect(),this.adapter_.addClass(l),this.activationTimer_=setTimeout(function(){return e.activationTimerCallback_()},u)},t.prototype.getFgTranslationCoordinates_=function(){var e,t=this.activationState_,n=t.activationEvent;return{startPoint:e={x:(e=t.wasActivatedByPointer?p(n,this.adapter_.getWindowPageOffset(),this.adapter_.computeBoundingRect()):{x:this.frame_.width/2,y:this.frame_.height/2}).x-this.initialSize_/2,y:e.y-this.initialSize_/2},endPoint:{x:this.frame_.width/2-this.initialSize_/2,y:this.frame_.height/2-this.initialSize_/2}}},t.prototype.runDeactivationUXLogicIfReady_=function(){var e=this,n=t.cssClasses.FG_DEACTIVATION,r=this.activationState_,o=r.hasDeactivationUXRun,i=r.isActivated;(o||!i)&&this.activationAnimationHasEnded_&&(this.rmBoundedActivationClasses_(),this.adapter_.addClass(n),this.fgDeactivationRemovalTimer_=setTimeout(function(){e.adapter_.removeClass(n)},f.FG_DEACTIVATION_MS))},t.prototype.rmBoundedActivationClasses_=function(){var e=t.cssClasses.FG_ACTIVATION;this.adapter_.removeClass(e),this.activationAnimationHasEnded_=!1,this.adapter_.computeBoundingRect()},t.prototype.resetActivationState_=function(){var e=this;this.previousActivationEvent_=this.activationState_.activationEvent,this.activationState_=this.defaultActivationState_(),setTimeout(function(){return e.previousActivationEvent_=void 0},t.numbers.TAP_DELAY_MS)},t.prototype.deactivate_=function(){var e=this,t=this.activationState_;if(t.isActivated){var n=l.a({},t);t.isProgrammatic?(requestAnimationFrame(function(){return e.animateDeactivation_(n)}),this.resetActivationState_()):(this.deregisterDeactivationHandlers_(),requestAnimationFrame(function(){e.activationState_.hasDeactivationUXRun=!0,e.animateDeactivation_(n),e.resetActivationState_()}))}},t.prototype.animateDeactivation_=function(e){var t=e.wasActivatedByPointer,n=e.wasElementMadeActive;(t||n)&&this.runDeactivationUXLogicIfReady_()},t.prototype.layoutInternal_=function(){var e=this;this.frame_=this.adapter_.computeBoundingRect();var n=Math.max(this.frame_.height,this.frame_.width);this.maxRadius_=this.adapter_.isUnbounded()?n:Math.sqrt(Math.pow(e.frame_.width,2)+Math.pow(e.frame_.height,2))+t.numbers.PADDING,this.initialSize_=Math.floor(n*t.numbers.INITIAL_ORIGIN_SCALE),this.fgScale_=""+this.maxRadius_/this.initialSize_,this.updateLayoutCssVars_()},t.prototype.updateLayoutCssVars_=function(){var e=t.strings,n=e.VAR_FG_SIZE,r=e.VAR_LEFT,o=e.VAR_TOP,i=e.VAR_FG_SCALE;this.adapter_.updateCssVariable(n,this.initialSize_+"px"),this.adapter_.updateCssVariable(i,this.fgScale_),this.adapter_.isUnbounded()&&(this.unboundedCoords_={left:Math.round(this.frame_.width/2-this.initialSize_/2),top:Math.round(this.frame_.height/2-this.initialSize_/2)},this.adapter_.updateCssVariable(r,this.unboundedCoords_.left+"px"),this.adapter_.updateCssVariable(o,this.unboundedCoords_.top+"px"))},t}(u),b=n(98);function _(e,t){if(void 0===e&&(e=window),void 0===t&&(t=!1),void 0===h||t){var n=!1;try{e.document.addEventListener("test",function(){},{get passive(){return n=!0}})}catch(r){}h=n}return!!h&&{passive:!0}}var E=n(2),T=n(36),C=n(13),O=n(23);n.d(t,"a",function(){return k}),n.d(t,"b",function(){return P});var w=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(t,n)};return function(t,n){function r(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(r.prototype=n.prototype,new r)}}(),S=function(){return(S=Object.assign||function(e){for(var t,n=1,r=arguments.length;n=(i=(m+v)/2))?m=i:v=i,(c=n>=(a=(y+g)/2))?y=a:g=a,o=p,!(p=p[f=c<<1|s]))return o[f]=h,e;if(l=+e._x.call(null,p.data),u=+e._y.call(null,p.data),t===l&&n===u)return h.next=p,o?o[f]=h:e._root=h,e;do{o=o?o[f]=new Array(4):e._root=new Array(4),(s=t>=(i=(m+v)/2))?m=i:v=i,(c=n>=(a=(y+g)/2))?y=a:g=a}while((f=c<<1|s)===(d=(u>=a)<<1|l>=i));return o[d]=p,o[f]=h,e}var i=function(e,t,n,r,o){this.node=e,this.x0=t,this.y0=n,this.x1=r,this.y1=o};function a(e){return e[0]}function l(e){return e[1]}function u(e,t,n){var r=new s(null==t?a:t,null==n?l:n,NaN,NaN,NaN,NaN);return null==e?r:r.addAll(e)}function s(e,t,n,r,o,i){this._x=e,this._y=t,this._x0=n,this._y0=r,this._x1=o,this._y1=i,this._root=void 0}function c(e){for(var t={data:e.data},n=t;e=e.next;)n=n.next={data:e.data};return t}var f=u.prototype=s.prototype;f.copy=function(){var e,t,n=new s(this._x,this._y,this._x0,this._y0,this._x1,this._y1),r=this._root;if(!r)return n;if(!r.length)return n._root=c(r),n;for(e=[{source:r,target:n._root=new Array(4)}];r=e.pop();)for(var o=0;o<4;++o)(t=r.source[o])&&(t.length?e.push({source:t,target:r.target[o]=new Array(4)}):r.target[o]=c(t));return n},f.add=function(e){var t=+this._x.call(null,e),n=+this._y.call(null,e);return o(this.cover(t,n),t,n,e)},f.addAll=function(e){var t,n,r,i,a=e.length,l=new Array(a),u=new Array(a),s=1/0,c=1/0,f=-1/0,d=-1/0;for(n=0;nf&&(f=r),id&&(d=i));if(s>f||c>d)return this;for(this.cover(s,c).cover(f,d),n=0;ne||e>=o||r>t||t>=i;)switch(l=(tp||(a=s.y0)>h||(l=s.x1)=g)<<1|e>=v)&&(s=m[m.length-1],m[m.length-1]=m[m.length-1-c],m[m.length-1-c]=s)}else{var b=e-+this._x.call(null,y.data),_=t-+this._y.call(null,y.data),E=b*b+_*_;if(E=(l=(h+y)/2))?h=l:y=l,(c=a>=(u=(m+v)/2))?m=u:v=u,t=p,!(p=p[f=c<<1|s]))return this;if(!p.length)break;(t[f+1&3]||t[f+2&3]||t[f+3&3])&&(n=t,d=f)}for(;p.data!==e;)if(r=p,!(p=p.next))return this;return(o=p.next)&&delete p.next,r?(o?r.next=o:delete r.next,this):t?(o?t[f]=o:delete t[f],(p=t[0]||t[1]||t[2]||t[3])&&p===(t[3]||t[2]||t[1]||t[0])&&!p.length&&(n?n[d]=p:this._root=p),this):(this._root=o,this)},f.removeAll=function(e){for(var t=0,n=e.length;t=0&&(n=e.slice(r+1),e=e.slice(0,r)),e&&!t.hasOwnProperty(e))throw new Error("unknown type: "+e);return{type:e,name:n}})}function E(e,t){for(var n,r=0,o=e.length;r0)for(var n,r,o=new Array(n),i=0;i=0&&t._call.call(null,e),t=t._next;--S}()}finally{S=0,function(){var e,t,n=C,r=1/0;for(;n;)n._call?(r>n._time&&(r=n._time),e=n,n=n._next):(t=n._next,n._next=null,n=e?e._next=t:C=t);O=e,V(r)}(),P=0}}function B(){var e=L.now(),t=e-k;t>A&&(N-=t,k=e)}function V(e){S||(x&&(x=clearTimeout(x)),e-P>24?(e<1/0&&(x=setTimeout(U,e-L.now()-N)),I&&(I=clearInterval(I))):(I||(k=L.now(),I=setInterval(B,A)),S=1,R(U)))}j.prototype=F.prototype={constructor:j,restart:function(e,t,n){if("function"!==typeof e)throw new TypeError("callback is not a function");n=(null==n?D():+n)+(null==t?0:+t),this._next||O===this||(O?O._next=this:C=this,O=this),this._call=e,this._time=n,V()},stop:function(){this._call&&(this._call=null,this._time=1/0,V())}};function H(e){return e.x}function z(e){return e.y}var K=10,W=Math.PI*(3-Math.sqrt(5)),G=function(e){var t,n=1,r=.001,o=1-Math.pow(r,1/300),i=0,a=.6,l=new Map,u=F(c),s=w("tick","end");function c(){f(),s.call("tick",t),n1?(null==n?l.delete(e):l.set(e,p(n)),t):l.get(e)},find:function(t,n,r){var o,i,a,l,u,s=0,c=e.length;for(null==r?r=1/0:r*=r,s=0;s1?(s.on(e,n),t):s.on(e)}}},q=function(){var e,t,n,r,o=d(-30),i=1,a=1/0,l=.81;function s(r){var o,i=e.length,a=u(e,H,z).visitAfter(f);for(n=r,o=0;o=a)){(e.data!==t||e.next)&&(0===c&&(h+=(c=p())*c),0===f&&(h+=(f=p())*f),h0;this.notchOutline(e)},t.prototype.handleMenuOpened=function(){this.adapter_.addClass(c.ACTIVATED)},t.prototype.handleMenuClosed=function(){this.adapter_.removeClass(c.ACTIVATED)},t.prototype.handleChange=function(e){void 0===e&&(e=!0);var t=this.getValue(),n=t.length>0,r=this.adapter_.hasClass(c.REQUIRED);this.notchOutline(n),this.adapter_.hasClass(c.FOCUSED)||this.adapter_.floatLabel(n),e&&(this.adapter_.notifyChange(t),r&&(this.setValid(this.isValid()),this.helperText_&&this.helperText_.setValidity(this.isValid())))},t.prototype.handleFocus=function(){this.adapter_.addClass(c.FOCUSED),this.adapter_.floatLabel(!0),this.notchOutline(!0),this.adapter_.activateBottomLine(),this.helperText_&&this.helperText_.showToScreenReader()},t.prototype.handleBlur=function(){this.adapter_.isMenuOpen()||(this.adapter_.removeClass(c.FOCUSED),this.handleChange(!1),this.adapter_.deactivateBottomLine(),this.adapter_.hasClass(c.REQUIRED)&&(this.setValid(this.isValid()),this.helperText_&&this.helperText_.setValidity(this.isValid())))},t.prototype.handleClick=function(e){this.adapter_.isMenuOpen()||(this.adapter_.setRippleCenter(e),this.adapter_.openMenu())},t.prototype.handleKeydown=function(e){if(!this.adapter_.isMenuOpen()){var t="Enter"===e.key||13===e.keyCode,n="Space"===e.key||32===e.keyCode,r="ArrowUp"===e.key||38===e.keyCode,o="ArrowDown"===e.key||40===e.keyCode;this.adapter_.hasClass(c.FOCUSED)&&(t||n||r||o)&&(this.adapter_.openMenu(),e.preventDefault())}},t.prototype.notchOutline=function(e){if(this.adapter_.hasOutline()){var t=this.adapter_.hasClass(c.FOCUSED);if(e){var n=d.LABEL_SCALE,r=this.adapter_.getLabelWidth()*n;this.adapter_.notchOutline(r)}else t||this.adapter_.closeOutline()}},t.prototype.setLeadingIconAriaLabel=function(e){this.leadingIcon_&&this.leadingIcon_.setAriaLabel(e)},t.prototype.setLeadingIconContent=function(e){this.leadingIcon_&&this.leadingIcon_.setContent(e)},t.prototype.setValid=function(e){this.adapter_.setValid(e)},t.prototype.isValid=function(){return this.adapter_.checkValidity()},t}(s),h={ICON_EVENT:"MDCSelect:icon",ICON_ROLE:"button"},m=["click","keydown"],y=function(e){function t(n){var r=e.call(this,u.a({},t.defaultAdapter,n))||this;return r.savedTabIndex_=null,r.interactionHandler_=function(e){return r.handleInteraction(e)},r}return u.b(t,e),Object.defineProperty(t,"strings",{get:function(){return h},enumerable:!0,configurable:!0}),Object.defineProperty(t,"defaultAdapter",{get:function(){return{getAttr:function(){return null},setAttr:function(){},removeAttr:function(){},setContent:function(){},registerInteractionHandler:function(){},deregisterInteractionHandler:function(){},notifyIconAction:function(){}}},enumerable:!0,configurable:!0}),t.prototype.init=function(){var e=this;this.savedTabIndex_=this.adapter_.getAttr("tabindex"),m.forEach(function(t){e.adapter_.registerInteractionHandler(t,e.interactionHandler_)})},t.prototype.destroy=function(){var e=this;m.forEach(function(t){e.adapter_.deregisterInteractionHandler(t,e.interactionHandler_)})},t.prototype.setDisabled=function(e){this.savedTabIndex_&&(e?(this.adapter_.setAttr("tabindex","-1"),this.adapter_.removeAttr("role")):(this.adapter_.setAttr("tabindex",this.savedTabIndex_),this.adapter_.setAttr("role",h.ICON_ROLE)))},t.prototype.setAriaLabel=function(e){this.adapter_.setAttr("aria-label",e)},t.prototype.setContent=function(e){this.adapter_.setContent(e)},t.prototype.handleInteraction=function(e){var t="Enter"===e.key||13===e.keyCode;("click"===e.type||t)&&this.adapter_.notifyIconAction()},t}(s),v=n(6),g=n(100),b=n(36),_=function(){function e(e){void 0===e&&(e={}),this.adapter_=e}return Object.defineProperty(e,"cssClasses",{get:function(){return{}},enumerable:!0,configurable:!0}),Object.defineProperty(e,"strings",{get:function(){return{}},enumerable:!0,configurable:!0}),Object.defineProperty(e,"numbers",{get:function(){return{}},enumerable:!0,configurable:!0}),Object.defineProperty(e,"defaultAdapter",{get:function(){return{}},enumerable:!0,configurable:!0}),e.prototype.init=function(){},e.prototype.destroy=function(){},e}(),E={LABEL_FLOAT_ABOVE:"mdc-floating-label--float-above",LABEL_SHAKE:"mdc-floating-label--shake",ROOT:"mdc-floating-label"},T=function(e){function t(n){var r=e.call(this,u.a({},t.defaultAdapter,n))||this;return r.shakeAnimationEndHandler_=function(){return r.handleShakeAnimationEnd_()},r}return u.b(t,e),Object.defineProperty(t,"cssClasses",{get:function(){return E},enumerable:!0,configurable:!0}),Object.defineProperty(t,"defaultAdapter",{get:function(){return{addClass:function(){},removeClass:function(){},getWidth:function(){return 0},registerInteractionHandler:function(){},deregisterInteractionHandler:function(){}}},enumerable:!0,configurable:!0}),t.prototype.init=function(){this.adapter_.registerInteractionHandler("animationend",this.shakeAnimationEndHandler_)},t.prototype.destroy=function(){this.adapter_.deregisterInteractionHandler("animationend",this.shakeAnimationEndHandler_)},t.prototype.getWidth=function(){return this.adapter_.getWidth()},t.prototype.shake=function(e){var n=t.cssClasses.LABEL_SHAKE;e?this.adapter_.addClass(n):this.adapter_.removeClass(n)},t.prototype.float=function(e){var n=t.cssClasses,r=n.LABEL_FLOAT_ABOVE,o=n.LABEL_SHAKE;e?this.adapter_.addClass(r):(this.adapter_.removeClass(r),this.adapter_.removeClass(o))},t.prototype.handleShakeAnimationEnd_=function(){var e=t.cssClasses.LABEL_SHAKE;this.adapter_.removeClass(e)},t}(_),C=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(t,n)};return function(t,n){function r(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(r.prototype=n.prototype,new r)}}(),O=function(){return(O=Object.assign||function(e){for(var t,n=1,r=arguments.length;n0&&(e+=j.NOTCH_ELEMENT_PADDING),this.adapter_.setNotchWidthProperty(e),this.adapter_.addClass(n)},t.prototype.closeNotch=function(){var e=t.cssClasses.OUTLINE_NOTCHED;this.adapter_.removeClass(e),this.adapter_.removeNotchWidthProperty()},t}(D),B=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(t,n)};return function(t,n){function r(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(r.prototype=n.prototype,new r)}}(),V=function(){return(V=Object.assign||function(e){for(var t,n=1,r=arguments.length;nY.ANCHOR_TO_MENU_SURFACE_WIDTH_RATIO&&(i="center"),(this.isHoistedElement_||this.isFixedPosition_)&&this.adjustPositionForHoistedElement_(f),this.adapter_.setTransformOrigin(i+" "+r),this.adapter_.setPosition(f),this.adapter_.setMaxHeight(n?n+"px":"")},t.prototype.getAutoLayoutMeasurements_=function(){var e=this.adapter_.getAnchorDimensions(),t=this.adapter_.getBodyDimensions(),n=this.adapter_.getWindowDimensions(),r=this.adapter_.getWindowScroll();return e||(e={top:this.position_.y,right:this.position_.x,bottom:this.position_.y,left:this.position_.x,width:0,height:0}),{anchorSize:e,bodySize:t,surfaceSize:this.dimensions_,viewportDistance:{top:e.top,right:n.width-e.right,bottom:n.height-e.bottom,left:e.left},viewportSize:n,windowScroll:r}},t.prototype.getOriginCorner_=function(){var e=i.TOP_LEFT,t=this.measurements_,n=t.viewportDistance,r=t.anchorSize,a=t.surfaceSize,l=this.hasBit_(this.anchorCorner_,o.BOTTOM),u=l?n.top+r.height+this.anchorMargin_.bottom:n.top+this.anchorMargin_.top,s=l?n.bottom-this.anchorMargin_.bottom:n.bottom+r.height-this.anchorMargin_.top,c=a.height-u,f=a.height-s;f>0&&c0&&g=0&&(this.adapter_.removeAttributeFromElementAtIndex(t,J.ARIA_CHECKED_ATTR),this.adapter_.removeClassFromElementAtIndex(t,Z.MENU_SELECTED_LIST_ITEM)),this.adapter_.addClassToElementAtIndex(e,Z.MENU_SELECTED_LIST_ITEM),this.adapter_.addAttributeToElementAtIndex(e,J.ARIA_CHECKED_ATTR,"true")},t.prototype.validatedIndex_=function(e){var t=this.adapter_.getMenuItemCount();if(!(e>=0&&e0?t[0]:null,this.lastFocusableElement=t.length>0?t[t.length-1]:null,this.foundation.open()}else this.foundation&&this.foundation.isOpen()&&this.foundation.close()},enumerable:!0,configurable:!0}),t.prototype.getDefaultFoundation=function(){var e=this;return new $(ue({addClass:function(t){e.root.addClass(t)},removeClass:function(t){e.root.removeClass(t)},hasClass:function(t){return"mdc-menu-surface"===t||e.root.hasClass(t)},hasAnchor:function(){return!!e.anchorElement},notifyClose:function(){e.emit("onClose",{}),e.deregisterBodyClickListener(),e.props.open&&(e.open=e.props.open)},notifyOpen:function(){e.emit("onOpen",{}),e.registerBodyClickListener()},isElementInContainer:function(t){return e.root.ref===t||!!e.root.ref&&e.root.ref.contains(t)},isRtl:function(){return!!e.root.ref&&"rtl"===getComputedStyle(e.root.ref).getPropertyValue("direction")},setTransformOrigin:function(t){e.root.setStyle(r.getTransformPropertyName(window)+"-origin",t)}},this.getFocusAdapterMethods(),this.getDimensionAdapterMethods()))},t.prototype.getFocusAdapterMethods=function(){var e=this;return{isFocused:function(){return document.activeElement===e.root.ref},saveFocus:function(){e.previousFocus=document.activeElement},restoreFocus:function(){e.root.ref&&e.root.ref.contains(document.activeElement)&&e.previousFocus&&e.previousFocus.focus&&e.previousFocus.focus()},isFirstElementFocused:function(){return!!e.firstFocusableElement&&e.firstFocusableElement===document.activeElement},isLastElementFocused:function(){return!!e.firstFocusableElement&&e.firstFocusableElement===document.activeElement},focusFirstElement:function(){return!!e.firstFocusableElement&&e.firstFocusableElement.focus&&e.firstFocusableElement.focus()},focusLastElement:function(){return!!e.firstFocusableElement&&e.firstFocusableElement.focus&&e.firstFocusableElement.focus()}}},t.prototype.getDimensionAdapterMethods=function(){var e=this;return{getInnerDimensions:function(){return{width:e.root.ref?e.root.ref.offsetWidth:0,height:e.root.ref?e.root.ref.offsetHeight:0}},getAnchorDimensions:function(){return e.anchorElement&&e.anchorElement.getBoundingClientRect()},getWindowDimensions:function(){return{width:window.innerWidth,height:window.innerHeight}},getBodyDimensions:function(){return{width:document.body.clientWidth,height:document.body.clientHeight}},getWindowScroll:function(){return{x:window.pageXOffset,y:window.pageYOffset}},setPosition:function(t){e.root.setStyle("left",void 0!==t.left?t.left:null),e.root.setStyle("right",void 0!==t.right?t.right:null),e.root.setStyle("top",void 0!==t.top?t.top:null),e.root.setStyle("bottom",void 0!==t.bottom?t.bottom:null)},setMaxHeight:function(t){e.root.setStyle("maxHeight",t)}}},t.prototype.sync=function(e,t){var n=this;this.syncProp(e.fixed,t.fixed,function(){n.foundation.setFixedPosition(!!e.fixed)}),this.syncProp(e.hoistToBody,t.hoistToBody,function(){e.hoistToBody?n.hoistMenuToBody():n.unhoistMenuFromBody()});var r=e.anchorCorner&&function(e){return $.Corner[ce[e]]}(e.anchorCorner);this.syncProp(r,this.foundation.anchorCorner_,function(){r&&(n.foundation.setAnchorCorner(r),n.foundation.dimensions_=n.foundation.adapter_.getInnerDimensions(),n.foundation.autoPosition_())}),this.syncProp(e.open,t.open,function(){n.open=!!e.open})},t.prototype.hoistMenuToBody=function(){var e=this;this.root.ref&&this.root.ref.parentElement&&(document.body.appendChild(this.root.ref.parentElement.removeChild(this.root.ref)),this.hoisted=!0,this.foundation.setIsHoisted(!0),this.props.open&&setTimeout(function(){return e.foundation.autoPosition_()}))},t.prototype.unhoistMenuFromBody=function(){this.anchorElement&&this.root.ref&&(this.anchorElement.appendChild(this.root.ref),this.hoisted=!1,this.foundation.setIsHoisted(!1))},t.prototype.setAnchorCorner=function(e){this.foundation.setAnchorCorner(e)},t.prototype.registerBodyClickListener=function(){document.body.addEventListener("click",this.handleBodyClick),document.body.addEventListener("touchstart",this.handleBodyClick)},t.prototype.deregisterBodyClickListener=function(){document.body.removeEventListener("click",this.handleBodyClick),document.body.removeEventListener("touchstart",this.handleBodyClick)},t.prototype.handleBodyClick=function(e){this.foundation&&this.foundation.handleBodyClick(e)},t.prototype.handleKeydown=function(e){this.props.onKeyDown&&this.props.onKeyDown(e),this.foundation.handleKeydown(e)},t.prototype.render=function(){var e=this.props,t=e.children,n=(e.open,e.anchorCorner,e.onOpen,e.onClose,e.hoistToBody,se(e,["children","open","anchorCorner","onOpen","onClose","hoistToBody"]));return a.createElement(fe,ue({},this.root.props(n),{ref:this.root.setRef,onKeyDown:this.handleKeydown}),t)},t.displayName="MenuSurface",t}(b.a),pe=Object(v.a)({displayName:"MenuSurfaceAnchor",classNames:["mdc-menu-surface--anchor"]}),he=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(t,n)};return function(t,n){function r(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(r.prototype=n.prototype,new r)}}(),me=function(){return(me=Object.assign||function(e){for(var t,n=1,r=arguments.length;n0&&!t.some(function(e){return e===document.activeElement})&&t[0].focus(),this.props.onOpen&&this.props.onOpen(e)},t.prototype.render=function(){var e=this,t=this.props,n=t.children,r=(t.focusOnOpen,ye(t,["children","focusOnOpen"])),o=!(n&&"object"===typeof n&&"MenuItems"===(n.type||{}).displayName);return a.createElement(de,me({},r,{"aria-hidden":!r.open,className:"mdc-menu "+(r.className||""),onKeyDown:this.handleKeydown,onClick:this.handleClick,onOpen:this.handleOpen,ref:function(t){return e.menuSurface=t}}),o?a.createElement(ve,{ref:function(t){return e.list=t}},n):a.cloneElement(n,{ref:function(t){return e.list=t}}))},t.displayName="Menu",t.defaultProps={focusOnOpen:!0},t}(b.a),_e=function(e){var t;return(t=function(t){function n(){var e=null!==t&&t.apply(this,arguments)||this;return e.state={open:!!e.props.open},e}return he(n,t),n.prototype.componentDidMount=function(){this.syncWithOpenProp(this.props.open)},n.prototype.componentDidUpdate=function(e){this.syncWithOpenProp(e.open)},n.prototype.syncWithOpenProp=function(e){void 0!==e&&this.state.open!==e&&this.setState({open:e})},n.prototype.render=function(){var t=this,n=this.props,r=n.handle,o=n.onClose,i=n.children,l=n.rootProps,u=void 0===l?{}:l,s=n.open,c=ye(n,["handle","onClose","children","rootProps","open"]),f=a.cloneElement(r,me({},r.props,{onClick:function(e){t.setState({open:!t.state.open}),r.props.onClick&&r.props.onClick(e)}}));return a.createElement(pe,me({},u),a.createElement(e,me({},c,{onClose:function(e){t.setState({open:!!s||!1}),o&&o(e)},open:this.state.open}),i),f)},n}(a.Component)).displayName="Simple"+e.displayName,t},Ee=(_e(be),_e(de),n(7));n.d(t,"a",function(){return Ne});var Te=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(t,n)};return function(t,n){function r(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(r.prototype=n.prototype,new r)}}(),Ce=function(){return(Ce=Object.assign||function(e){for(var t,n=1,r=arguments.length;n=0&&this.menu.items[this.state.selectedIndex].focus()},t.prototype.handleMenuClosed=function(){this.setState({menuOpen:!1}),document.activeElement!==this.selectedText&&this.foundation.handleBlur()},t.prototype.renderIcon=function(e,t){var n=this;return e&&"string"===typeof e||e.type&&e.type.displayName!==ke.displayName?a.createElement(ke,{ref:function(e){"leadingIcon_"===t?n.leadingIcon_=e&&e.foundation:n.trailingIcon_=e&&e.foundation},tabIndex:"trailingIcon_"===t?0:void 0,icon:e}):e},t.prototype.renderHelpText=function(){var e=this.props.helpText;if(!!!e)return null;var t="object"===typeof e&&!a.isValidElement(e);return e&&t?a.createElement(Pe,Ce({},e)):a.createElement(Pe,null,e)},t.prototype.render=function(){var e=this,t=this.props,n=t.placeholder,r=t.children,o=t.value,i=t.outlined,u=t.label,s=void 0===u?"":u,c=t.options,f=void 0===c?[]:c,d=t.rootProps,p=void 0===d?{}:d,h=t.className,m=t.enhanced,y=(t.icon,t.withLeadingIcon,t.onChange,t.onFocus,t.onBlur,t.onKeyDown,t.invalid),v=t.inputRef,g=(t.helpText,Oe(t,["placeholder","children","value","outlined","label","options","rootProps","className","enhanced","icon","withLeadingIcon","onChange","onFocus","onBlur","onKeyDown","invalid","inputRef","helpText"])),b=this.props,_=b.icon,E=b.withLeadingIcon;void 0!==E&&(Object(l.a)("Select prop 'withLeadingIcon' is now 'icon'."),_=E);var T=function e(t){return Array.isArray(t)&&t[0]&&"object"===typeof t[0]?t.map(function(t){if("object"!==typeof t)throw new Error("Encountered non object for Select "+t);return Ce({},t,{options:e(t.options)})}):Array.isArray(t)?t.map(function(e){return{value:e,label:e}}):"object"===typeof t?Object.keys(t).map(function(e){return{value:e,label:t[e]}}):t}(f),C=void 0!==o?void 0:this.props.defaultValue||"",O={onChange:this.handleChange,onFocus:this.handleFocus,onBlur:this.handleBlur,onTouchStart:this.handleClick,onMouseDown:this.handleClick},w={defaultValue:C,value:o,placeholder:n,selectOptions:T},x=a.createElement(S,Ce({},this.label.props({}),{ref:this.label.setRef}),s);return a.createElement(a.Fragment,null,a.createElement(we,Ce({ripple:!i},this.root.props(Ce({className:h},p)),{invalid:y,required:g.required,icon:_,outlined:i,ref:this.root.setRef}),!!_&&this.renderIcon(_,"leadingIcon_"),a.createElement(Se,null),m?a.createElement(a.Fragment,null,a.createElement("input",{type:"hidden",ref:function(t){return e.hiddenInput_=t}}),a.createElement("div",Ce({ref:function(t){return e.selectedText=t},className:"mdc-select__selected-text",tabIndex:this.props.disabled?-1:0,"aria-disabled":this.props.disabled?"true":"false","aria-expanded":this.state.menuOpen,onKeyDown:this.handleKeydown},O),this.state.selectedTextContent),a.createElement(Ie,Ce({},w,{selectedIndex:this.state.selectedIndex,apiRef:function(t){e.menu=t},open:this.state.menuOpen,onClose:this.handleMenuClosed,onOpen:this.handleMenuOpened,onSelect:this.handleMenuSelected}),r)):a.createElement(xe,Ce({},g,{elementRef:function(t){e.nativeControl=t,v&&v(t)}},w,O),r),i?a.createElement(W,Ce({},this.outline.props({})),x):a.createElement(a.Fragment,null,x,a.createElement(L,Ce({},this.lineRipple.props({}))))),this.renderHelpText())},t}(b.a),ke=function(e){function t(){var t=null!==e&&e.apply(this,arguments)||this;return t.root=t.createElement("root"),t}return Te(t,e),t.prototype.getDefaultFoundation=function(){var e=this;return new y({getAttr:function(t){return e.root.getProp(t)},setAttr:function(t,n){return e.root.setProp(t,n)},removeAttr:function(t){return e.root.removeProp(t)},setContent:function(t){e.root.ref&&(e.root.ref.textContent=t)},registerInteractionHandler:function(t,n){return e.root.addEventListener(t,n)},deregisterInteractionHandler:function(t,n){return e.root.removeEventListener(t,n)},notifyIconAction:function(){return e.emit("onClick",{},!0)}})},t.prototype.render=function(){return a.createElement(R.a,Ce({},this.root.props(Ce({},this.props,{className:"mdc-select__icon"}))))},t.displayName="SelectIcon",t}(b.a),Pe=Object(v.a)({displayName:"SelectHelperText",tag:"p",classNames:function(e){return["mdc-select-helper-text",{"mdc-select-helper-text--persistent":e.persistent,"mdc-select-helper-text--validation-msg":e.validationMsg}]},consumeProps:["persistent","validationMsg"]}),Ne=function(e){var t=e.enhanced,n=Oe(e,["enhanced"]);return a.createElement(Ae,Ce({key:t?"enhanced":"native",enhanced:t},n))};Ne.displayName="Select"},,function(e,t,n){"use strict";var r=Object.getOwnPropertySymbols,o=Object.prototype.hasOwnProperty,i=Object.prototype.propertyIsEnumerable;function a(e){if(null===e||void 0===e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},r)).join("")}catch(o){return!1}}()?Object.assign:function(e,t){for(var n,l,u=a(e),s=1;s0)&&!(r=i.next()).done;)a.push(r.value)}catch(l){o={error:l}}finally{try{r&&!r.done&&(n=i.return)&&n.call(i)}finally{if(o)throw o.error}}return a},d=function(){for(var e=[],t=0;t0){var n=e[e.length-1];n!==t&&n.pause()}var r=e.indexOf(t);-1===r?e.push(t):(e.splice(r,1),e.push(t))},deactivateTrap:function(t){var n=e.indexOf(t);-1!==n&&e.splice(n,1),e.length>0&&e[e.length-1].unpause()}}}();function l(e){return setTimeout(e,0)}e.exports=function(e,t){var n=document,u="string"===typeof e?n.querySelector(e):e,s=i({returnFocusOnDeactivate:!0,escapeDeactivates:!0},t),c={firstTabbableNode:null,lastTabbableNode:null,nodeFocusedBeforeActivation:null,mostRecentlyFocusedNode:null,active:!1,paused:!1},f={activate:function(e){if(c.active)return;E(),c.active=!0,c.paused=!1,c.nodeFocusedBeforeActivation=n.activeElement;var t=e&&e.onActivate?e.onActivate:s.onActivate;t&&t();return p(),f},deactivate:d,pause:function(){if(c.paused||!c.active)return;c.paused=!0,h()},unpause:function(){if(!c.paused||!c.active)return;c.paused=!1,E(),p()}};return f;function d(e){if(c.active){clearTimeout(r),h(),c.active=!1,c.paused=!1,a.deactivateTrap(f);var t=e&&void 0!==e.onDeactivate?e.onDeactivate:s.onDeactivate;return t&&t(),(e&&void 0!==e.returnFocus?e.returnFocus:s.returnFocusOnDeactivate)&&l(function(){T(c.nodeFocusedBeforeActivation)}),f}}function p(){if(c.active)return a.activateTrap(f),r=l(function(){T(y())}),n.addEventListener("focusin",g,!0),n.addEventListener("mousedown",v,{capture:!0,passive:!1}),n.addEventListener("touchstart",v,{capture:!0,passive:!1}),n.addEventListener("click",_,{capture:!0,passive:!1}),n.addEventListener("keydown",b,{capture:!0,passive:!1}),f}function h(){if(c.active)return n.removeEventListener("focusin",g,!0),n.removeEventListener("mousedown",v,!0),n.removeEventListener("touchstart",v,!0),n.removeEventListener("click",_,!0),n.removeEventListener("keydown",b,!0),f}function m(e){var t=s[e],r=t;if(!t)return null;if("string"===typeof t&&!(r=n.querySelector(t)))throw new Error("`"+e+"` refers to no known node");if("function"===typeof t&&!(r=t()))throw new Error("`"+e+"` did not return a node");return r}function y(){var e;if(!(e=null!==m("initialFocus")?m("initialFocus"):u.contains(n.activeElement)?n.activeElement:c.firstTabbableNode||m("fallbackFocus")))throw new Error("You can't have a focus-trap without at least one focusable element");return e}function v(e){u.contains(e.target)||(s.clickOutsideDeactivates?d({returnFocus:!o.isFocusable(e.target)}):s.allowOutsideClick&&s.allowOutsideClick(e)||e.preventDefault())}function g(e){u.contains(e.target)||e.target instanceof Document||(e.stopImmediatePropagation(),T(c.mostRecentlyFocusedNode||y()))}function b(e){if(!1!==s.escapeDeactivates&&function(e){return"Escape"===e.key||"Esc"===e.key||27===e.keyCode}(e))return e.preventDefault(),void d();(function(e){return"Tab"===e.key||9===e.keyCode})(e)&&function(e){if(E(),e.shiftKey&&e.target===c.firstTabbableNode)return e.preventDefault(),void T(c.lastTabbableNode);if(!e.shiftKey&&e.target===c.lastTabbableNode)e.preventDefault(),T(c.firstTabbableNode)}(e)}function _(e){s.clickOutsideDeactivates||u.contains(e.target)||s.allowOutsideClick&&s.allowOutsideClick(e)||(e.preventDefault(),e.stopImmediatePropagation())}function E(){var e=o(u);c.firstTabbableNode=e[0]||y(),c.lastTabbableNode=e[e.length-1]||y()}function T(e){e!==n.activeElement&&(e&&e.focus?(e.focus(),c.mostRecentlyFocusedNode=e,function(e){return e.tagName&&"input"===e.tagName.toLowerCase()&&"function"===typeof e.select}(e)&&e.select()):T(y()))}}},function(e,t,n){"use strict";var r=n(0);t.a=function(){var e=(arguments.length>0&&void 0!==arguments[0]?arguments[0]:{}).liveMeasure,t=void 0===e||e,n=Object(r.useState)({}),o=n[0],i=n[1],a=Object(r.useState)(null),l=a[0],u=a[1],s=Object(r.useCallback)(function(e){u(e)},[]);return Object(r.useLayoutEffect)(function(){if(l){var e=function(){return window.requestAnimationFrame(function(){return i(function(e){var t=e.getBoundingClientRect();return{width:t.width,height:t.height,top:"x"in t?t.x:t.top,left:"y"in t?t.y:t.left,x:"x"in t?t.x:t.left,y:"y"in t?t.y:t.top,right:t.right,bottom:t.bottom}}(l))})};if(e(),t)return window.addEventListener("resize",e),window.addEventListener("scroll",e),function(){window.removeEventListener("resize",e),window.removeEventListener("scroll",e)}}},[l]),[s,o,l]}},,function(e,t,n){"use strict";var r=n(0),o=n(6),i=n(13),a=n(7),l=n(14),u=n(1),s=function(){function e(e){void 0===e&&(e={}),this.adapter_=e}return Object.defineProperty(e,"cssClasses",{get:function(){return{}},enumerable:!0,configurable:!0}),Object.defineProperty(e,"strings",{get:function(){return{}},enumerable:!0,configurable:!0}),Object.defineProperty(e,"numbers",{get:function(){return{}},enumerable:!0,configurable:!0}),Object.defineProperty(e,"defaultAdapter",{get:function(){return{}},enumerable:!0,configurable:!0}),e.prototype.init=function(){},e.prototype.destroy=function(){},e}(),c={ICON_BUTTON_ON:"mdc-icon-button--on",ROOT:"mdc-icon-button"},f={ARIA_PRESSED:"aria-pressed",CHANGE_EVENT:"MDCIconButtonToggle:change"},d=function(e){function t(n){return e.call(this,u.a({},t.defaultAdapter,n))||this}return u.b(t,e),Object.defineProperty(t,"cssClasses",{get:function(){return c},enumerable:!0,configurable:!0}),Object.defineProperty(t,"strings",{get:function(){return f},enumerable:!0,configurable:!0}),Object.defineProperty(t,"defaultAdapter",{get:function(){return{addClass:function(){},hasClass:function(){return!1},notifyChange:function(){},removeClass:function(){},setAttr:function(){}}},enumerable:!0,configurable:!0}),t.prototype.init=function(){this.adapter_.setAttr(f.ARIA_PRESSED,""+this.isOn())},t.prototype.handleClick=function(){this.toggle(),this.adapter_.notifyChange({isOn:this.isOn()})},t.prototype.isOn=function(){return this.adapter_.hasClass(c.ICON_BUTTON_ON)},t.prototype.toggle=function(e){void 0===e&&(e=!this.isOn()),e?this.adapter_.addClass(c.ICON_BUTTON_ON):this.adapter_.removeClass(c.ICON_BUTTON_ON),this.adapter_.setAttr(f.ARIA_PRESSED,""+e)},t}(s),p=n(36),h=n(5),m=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(t,n)};return function(t,n){function r(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(r.prototype=n.prototype,new r)}}(),y=function(){return(y=Object.assign||function(e){for(var t,n=1,r=arguments.length;n=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}n.d(t,"a",function(){return r})},function(e,t,n){"use strict";var r=n(8),o=n.n(r),i=n(0),a=n.n(i),l=n(3),u=n.n(l);function s(){return(s=Object.assign||function(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}function f(e){var t=e.top,n=void 0===t?0:t,r=e.left,i=void 0===r?0:r,l=e.transform,u=e.className,f=e.children,d=e.innerRef,p=c(e,["top","left","transform","className","children","innerRef"]);return a.a.createElement("g",s({ref:d,className:o()("vx-group",u),transform:l||"translate(".concat(i,", ").concat(n,")")},p),f)}function d(e){var t=e.links,n=e.linkComponent,r=e.className;return a.a.createElement(f,null,t.map(function(e,t){return a.a.createElement(f,{className:o()("vx-network-links",r),key:"network-link-".concat(t)},a.a.createElement(n,{link:e}))}))}function p(e){var t=e.nodes,n=e.nodeComponent,r=e.className;return a.a.createElement(f,null,t.map(function(e,t){return a.a.createElement(f,{key:"network-node-".concat(t),className:o()("vx-network-nodes",r),transform:"translate(".concat(e.x,", ").concat(e.y,")")},a.a.createElement(n,{node:e}))}))}function h(e){var t=e.link;return a.a.createElement("line",{x1:t.source.x,y1:t.source.y,x2:t.target.x,y2:t.target.y,strokeWidth:2,stroke:"#999",strokeOpacity:.6})}function m(){return a.a.createElement("circle",{r:15,fill:"#21D4FD"})}function y(e){var t=e.graph,n=e.linkComponent,r=void 0===n?h:n,o=e.nodeComponent,i=void 0===o?m:o;return a.a.createElement(f,null,a.a.createElement(d,{links:t.links,linkComponent:r}),a.a.createElement(p,{nodes:t.nodes,nodeComponent:i}))}f.propTypes={top:u.a.number,left:u.a.number,transform:u.a.string,className:u.a.string,children:u.a.any,innerRef:u.a.oneOfType([u.a.func,u.a.object])},n.d(t,"a",function(){return y}),d.propTypes={links:u.a.array,linkComponent:u.a.any,className:u.a.string},p.propTypes={nodes:u.a.array,nodeComponent:u.a.any,className:u.a.string},h.propTypes={link:u.a.object},y.propTypes={graph:u.a.object,linkComponent:u.a.any,nodeComponent:u.a.any}},,,,,,function(e,t,n){"use strict";var r=n(28),o="function"===typeof Symbol&&Symbol.for,i=o?Symbol.for("react.element"):60103,a=o?Symbol.for("react.portal"):60106,l=o?Symbol.for("react.fragment"):60107,u=o?Symbol.for("react.strict_mode"):60108,s=o?Symbol.for("react.profiler"):60114,c=o?Symbol.for("react.provider"):60109,f=o?Symbol.for("react.context"):60110,d=o?Symbol.for("react.forward_ref"):60112,p=o?Symbol.for("react.suspense"):60113,h=o?Symbol.for("react.suspense_list"):60120,m=o?Symbol.for("react.memo"):60115,y=o?Symbol.for("react.lazy"):60116;o&&Symbol.for("react.fundamental"),o&&Symbol.for("react.responder"),o&&Symbol.for("react.scope");var v="function"===typeof Symbol&&Symbol.iterator;function g(e){for(var t=e.message,n="https://reactjs.org/docs/error-decoder.html?invariant="+t,r=1;rL.length&&L.push(e)}function M(e,t,n){return null==e?0:function e(t,n,r,o){var l=typeof t;"undefined"!==l&&"boolean"!==l||(t=null);var u=!1;if(null===t)u=!0;else switch(l){case"string":case"number":u=!0;break;case"object":switch(t.$$typeof){case i:case a:u=!0}}if(u)return r(o,t,""===n?"."+j(t,0):n),1;if(u=0,n=""===n?".":n+":",Array.isArray(t))for(var s=0;s