Repository: awslabs/dynamodb-titan-storage-backend
Branch: master
Commit: 4d05028e1b59
Files: 156
Total size: 3.3 MB
Directory structure:
gitextract_7697_i97/
├── .gitignore
├── .travis.yml
├── LICENSE.txt
├── NOTICE.txt
├── README.md
├── buildspec.yml
├── checkstyle.xml
├── dynamodb-janusgraph-storage-backend-cfn.yaml
├── dynamodb-janusgraph-tables-multiple.yaml
├── dynamodb-janusgraph-tables-single.yaml
├── pom.xml
└── src/
├── main/
│ ├── java/
│ │ └── com/
│ │ ├── amazon/
│ │ │ └── janusgraph/
│ │ │ ├── diskstorage/
│ │ │ │ └── dynamodb/
│ │ │ │ ├── AbstractDynamoDbStore.java
│ │ │ │ ├── AwsStore.java
│ │ │ │ ├── BackendDataModel.java
│ │ │ │ ├── BackendNotFoundException.java
│ │ │ │ ├── BackendRuntimeException.java
│ │ │ │ ├── Client.java
│ │ │ │ ├── Constants.java
│ │ │ │ ├── DynamoDBStoreManager.java
│ │ │ │ ├── DynamoDbDelegate.java
│ │ │ │ ├── DynamoDbSingleRowStore.java
│ │ │ │ ├── DynamoDbStore.java
│ │ │ │ ├── DynamoDbStoreFactory.java
│ │ │ │ ├── DynamoDbStoreTransaction.java
│ │ │ │ ├── ExponentialBackoff.java
│ │ │ │ ├── Expression.java
│ │ │ │ ├── GetItemResultWrapper.java
│ │ │ │ ├── GetItemWorker.java
│ │ │ │ ├── JanusGraphConfigUtil.java
│ │ │ │ ├── ListTablesWorker.java
│ │ │ │ ├── MetricStore.java
│ │ │ │ ├── PaginatingTask.java
│ │ │ │ ├── QueryResultWrapper.java
│ │ │ │ ├── QueryWithLimitWorker.java
│ │ │ │ ├── QueryWorker.java
│ │ │ │ ├── TableNameDynamoDbStoreFactory.java
│ │ │ │ ├── builder/
│ │ │ │ │ ├── AbstractBuilder.java
│ │ │ │ │ ├── ConditionExpressionBuilder.java
│ │ │ │ │ ├── EntryBuilder.java
│ │ │ │ │ ├── FilterExpressionBuilder.java
│ │ │ │ │ ├── ItemBuilder.java
│ │ │ │ │ ├── KeyBuilder.java
│ │ │ │ │ ├── MultiUpdateExpressionBuilder.java
│ │ │ │ │ ├── SingleExpectedAttributeValueBuilder.java
│ │ │ │ │ └── SingleUpdateBuilder.java
│ │ │ │ ├── iterator/
│ │ │ │ │ ├── MultiRecordIterator.java
│ │ │ │ │ ├── MultiRowParallelScanInterpreter.java
│ │ │ │ │ ├── MultiRowSequentialScanInterpreter.java
│ │ │ │ │ ├── ParallelScanner.java
│ │ │ │ │ ├── ScanBackedKeyIterator.java
│ │ │ │ │ ├── ScanContext.java
│ │ │ │ │ ├── ScanContextInterpreter.java
│ │ │ │ │ ├── ScanSegmentWorker.java
│ │ │ │ │ ├── Scanner.java
│ │ │ │ │ ├── SequentialScanner.java
│ │ │ │ │ ├── SingleKeyRecordIterator.java
│ │ │ │ │ ├── SingleRowScanInterpreter.java
│ │ │ │ │ └── StaticRecordIterator.java
│ │ │ │ └── mutation/
│ │ │ │ ├── DeleteItemWorker.java
│ │ │ │ ├── MutateWorker.java
│ │ │ │ ├── SingleUpdateWithCleanupWorker.java
│ │ │ │ └── UpdateItemWorker.java
│ │ │ └── example/
│ │ │ └── MarvelGraphFactory.java
│ │ └── google/
│ │ └── common/
│ │ └── util/
│ │ └── concurrent/
│ │ └── RateLimiterCreator.java
│ └── resources/
│ └── META-INF/
│ └── marvel.csv
└── test/
├── java/
│ └── com/
│ ├── amazon/
│ │ └── janusgraph/
│ │ ├── ClientTest.java
│ │ ├── DynamoDbStoreTransactionTest.java
│ │ ├── GraphOfTheGodsTest.java
│ │ ├── MarvelTest.java
│ │ ├── ScenarioTests.java
│ │ ├── TestCiHeartbeat.java
│ │ ├── TestGraphUtil.java
│ │ ├── diskstorage/
│ │ │ └── dynamodb/
│ │ │ ├── AbstractDynamoDBIDAuthorityTest.java
│ │ │ ├── AbstractDynamoDBLogTest.java
│ │ │ ├── AbstractDynamoDBMultiWriteStoreTest.java
│ │ │ ├── AbstractDynamoDbStoreTest.java
│ │ │ ├── DynamoDBLockStoreTest.java
│ │ │ ├── DynamoDbDelegateTest.java
│ │ │ ├── MultiDynamoDBIDAuthorityTest.java
│ │ │ ├── MultiDynamoDBLogTest.java
│ │ │ ├── MultiDynamoDBMultiWriteStoreTest.java
│ │ │ ├── MultiDynamoDBStoreTest.java
│ │ │ ├── SingleDynamoDBIDAuthorityTest.java
│ │ │ ├── SingleDynamoDBLogTest.java
│ │ │ ├── SingleDynamoDBMultiWriteStoreTest.java
│ │ │ └── SingleDynamoDBStoreTest.java
│ │ ├── graphdb/
│ │ │ └── dynamodb/
│ │ │ ├── AbstractDynamoDBEventualGraphTest.java
│ │ │ ├── AbstractDynamoDBGraphConcurrentTest.java
│ │ │ ├── AbstractDynamoDBGraphIterativeTest.java
│ │ │ ├── AbstractDynamoDBGraphPerformanceMemoryTest.java
│ │ │ ├── AbstractDynamoDBGraphSpeedTest.java
│ │ │ ├── AbstractDynamoDBGraphTest.java
│ │ │ ├── AbstractDynamoDBOLAPTest.java
│ │ │ ├── AbstractDynamoDBOperationCountingTest.java
│ │ │ ├── AbstractDynamoDBPartitionGraphTest.java
│ │ │ ├── MultiDynamoDBEventualGraphTest.java
│ │ │ ├── MultiDynamoDBGraphConcurrentTest.java
│ │ │ ├── MultiDynamoDBGraphPerformanceMemoryTest.java
│ │ │ ├── MultiDynamoDBGraphSpeedTest.java
│ │ │ ├── MultiDynamoDBGraphTest.java
│ │ │ ├── MultiDynamoDBOLAPTest.java
│ │ │ ├── MultiDynamoDBOperationCountingTest.java
│ │ │ ├── MultiDynamoDBPartitionGraphTest.java
│ │ │ ├── SingleDynamoDBEventualGraphTest.java
│ │ │ ├── SingleDynamoDBGraphConcurrentTest.java
│ │ │ ├── SingleDynamoDBGraphPerformanceMemoryTest.java
│ │ │ ├── SingleDynamoDBGraphSpeedTest.java
│ │ │ ├── SingleDynamoDBGraphTest.java
│ │ │ ├── SingleDynamoDBOLAPTest.java
│ │ │ ├── SingleDynamoDBOperationCountingTest.java
│ │ │ ├── SingleDynamoDBPartitionGraphTest.java
│ │ │ └── TestCombination.java
│ │ ├── testcategory/
│ │ │ ├── GraphSimpleLogTestCategory.java
│ │ │ ├── IsolateMultiConcurrentGetSlice.java
│ │ │ ├── IsolateMultiConcurrentGetSliceAndMutate.java
│ │ │ ├── IsolateMultiEdgesExceedCacheSize.java
│ │ │ ├── IsolateMultiLargeJointIndexRetrieval.java
│ │ │ ├── IsolateMultiVertexCentricQuery.java
│ │ │ ├── IsolateRemainingTestsCategory.java
│ │ │ ├── IsolateSingleConcurrentGetSlice.java
│ │ │ ├── IsolateSingleConcurrentGetSliceAndMutate.java
│ │ │ ├── MultiDynamoDBGraphTestCategory.java
│ │ │ ├── MultiDynamoDBMultiWriteStoreTestCategory.java
│ │ │ ├── MultiDynamoDBOLAPTestCategory.java
│ │ │ ├── MultiDynamoDBStoreTestCategory.java
│ │ │ ├── MultiIdAuthorityLogStoreCategory.java
│ │ │ ├── MultipleItemTestCategory.java
│ │ │ ├── SingleDynamoDBGraphTestCategory.java
│ │ │ ├── SingleDynamoDBMultiWriteStoreTestCategory.java
│ │ │ ├── SingleDynamoDBOLAPTestCategory.java
│ │ │ ├── SingleDynamoDBStoreTestCategory.java
│ │ │ ├── SingleIdAuthorityLogStoreCategory.java
│ │ │ └── SingleItemTestCategory.java
│ │ └── testutils/
│ │ ├── CiHeartbeat.java
│ │ └── HeartbeatTimerTask.java
│ └── google/
│ └── common/
│ └── util/
│ └── concurrent/
│ └── RateLimiterCreatorTest.java
└── resources/
├── META-INF/
│ └── HotelTriples.txt
├── current-gen-instance-types
├── docker-compose.yml
├── dynamodb-janusgraph-docker/
│ └── Dockerfile
├── dynamodb-local-docker/
│ └── Dockerfile
├── dynamodb-local-docker.properties
├── dynamodb-local.properties
├── dynamodb.properties
├── get-recent-al-amis.sh
├── gremlin-server-local-docker.yaml
├── gremlin-server-local.yaml
├── gremlin-server-service.sh
├── gremlin-server.yaml
├── install-gremlin-server.sh
├── install-reqs.sh
├── janusgraph-0.2.0-hadoop2.zip.asc
├── log4j.properties
├── remote.yaml
├── titan-upgrade.properties
└── type-to-arch.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
/target/
/pom.xml.tag
/pom.xml.releaseBackup
/pom.xml.versionsBackup
/pom.xml.next
/release.properties
/dependency-reduced-pom.xml
/buildNumber.properties
/.mvn/timing.properties
/server
/elasticsearch
/*.iml
/.idea
/.project
/.classpath
/.settings
.DS_Store
*~
/src/test/resources/dynamodb-janusgraph-docker/*.zip
================================================
FILE: .travis.yml
================================================
language: java
sudo: required
dist: trusty
jdk:
- oraclejdk8
cache:
directories:
- "${HOME}/.m2"
env:
matrix:
#32 minutes
- MODULE="MultiStoreTest" CATEGORY="MultiDynamoDBStoreTestCategory"
#22 minutes
- MODULE="MultiVertexCentricQuery" CATEGORY="IsolateMultiVertexCentricQuery"
#17.5 minutes
- MODULE="SingleGraphTest" CATEGORY="SingleDynamoDBGraphTestCategory"
#17.3 minutes
- MODULE="SingleStoreTest" CATEGORY="SingleDynamoDBStoreTestCategory"
#16.5 minutes
- MODULE="MultiLargeJointIndexRetrieval" CATEGORY="IsolateMultiLargeJointIndexRetrieval"
#15.7 minutes
- MODULE="MultiOLAPTest" CATEGORY="MultiDynamoDBOLAPTestCategory"
#13.5 minutes
- MODULE="MultiGraphTest" CATEGORY="MultiDynamoDBGraphTestCategory"
#11.8 minutes
- MODULE="MultiEdgesExceedCacheSize" CATEGORY="IsolateMultiEdgesExceedCacheSize"
#12 minutes
- MODULE="SingleIdAuthorityLogStore" CATEGORY="SingleIdAuthorityLogStoreCategory"
#11 minutes
- MODULE="MultiIdAuthorityLogStore" CATEGORY="MultiIdAuthorityLogStoreCategory"
#9 minutes
- MODULE="SingleOLAPTest" CATEGORY="SingleDynamoDBOLAPTestCategory"
#8.7 minutes
- MODULE="MultiConcurrentGetSliceAndMutate" CATEGORY="IsolateMultiConcurrentGetSliceAndMutate"
#8.5 minutes
- MODULE="RemainingTestsCategory" CATEGORY="IsolateRemainingTestsCategory"
#6.5 minutes
- MODULE="MultiConcurrentGetSlice" CATEGORY="IsolateMultiConcurrentGetSlice"
#6.4 minutes
- MODULE="SingleConcurrentGetSliceAndMutate" CATEGORY="IsolateSingleConcurrentGetSliceAndMutate"
#4.8 minutes
- MODULE="SingleConcurrentGetSlice" CATEGORY="IsolateSingleConcurrentGetSlice"
#3.2
- MODULE="SingleMultiWriteStoreTestCategory" CATEGORY="SingleDynamoDBMultiWriteStoreTestCategory"
#2.8 minutes
- MODULE="MultiMultiWriteStoreTestCategory" CATEGORY="MultiDynamoDBMultiWriteStoreTestCategory"
#To be added
- MODULE="GraphSimpleLogTest" CATEGORY="GraphSimpleLogTestCategory"
addons:
apt:
packages:
- oracle-java8-installer
branches:
only:
- 1.0.0
- master
- janusgraph
script:
#build the matrix item
- mvn install -P integration-tests -Dgroups="com.amazon.janusgraph.testcategory.${CATEGORY}" -Dinclude.category="**/*.java"
#validating cloudformation templates requires credentials. This will be a manual step, unfortunately.
#- aws cloudformation validate-template --region us-west-2 --template-body `pwd | sed -e 's/\//\/\//g' -e 's/^/file:\//' -e 's/$/\/\/dynamodb-janusgraph-storage-backend-cfn\.yaml/'`
#- aws cloudformation validate-template --region us-west-2 --template-body `pwd | sed -e 's/\//\/\//g' -e 's/^/file:\//' -e 's/$/\/\/dynamodb-janusgraph-tables-single\.yaml/'`
#- aws cloudformation validate-template --region us-west-2 --template-body `pwd | sed -e 's/\//\/\//g' -e 's/^/file:\//' -e 's/$/\/\/dynamodb-janusgraph-tables-multiple\.yaml/'`
notifications:
email:
- amcp@amazon.co.jp
- johanjcbs@gmail.com
================================================
FILE: LICENSE.txt
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
---
Copyright (c) 2009-Infinity, TinkerPop [http://tinkerpop.com]
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the TinkerPop nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL TINKERPOP BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================
FILE: NOTICE.txt
================================================
Amazon DynamoDB Storage Backend for Titan
Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
This product includes software developed at
Amazon Web Services, Inc. (http://aws.amazon.com/),
Amazon Fulfillment Technologies (https://www.amazon.jobs/team/amazon-fulfillment-technologies-aft) and
Amazon Japan (https://www.amazon.jobs/location/tokyo-area-japan).
**********************
THIRD PARTY COMPONENTS
**********************
This software includes third party software subject to the following copyrights:
- Titan: Distributed Graph Database - Copyright 2012 and onwards Aurelius. Apache License 2.0.
- JanusGraph: Distributed Graph Database - Copyright 2017 and onwards. Apache License 2.0.
- Guava: Google Core Libraries for Java - Copyright 2010 and onwards Google, Inc. Apache License 2.0.
- rexster-service.sh - Copyright (c) 2009-Infinity, TinkerPop [http://tinkerpop.com]. BSD License.
The licenses for these third party components are included in LICENSE.txt.
================================================
FILE: README.md
================================================
# Amazon DynamoDB Storage Backend for JanusGraph
> JanusGraph: Distributed Graph Database is a scalable graph database optimized for
> storing and querying graphs containing hundreds of billions of vertices and
> edges distributed across a multi-machine cluster. JanusGraph is a transactional
> database that can support thousands of concurrent users executing complex
> graph traversals in real time. -- [JanusGraph Homepage](http://janusgraph.org/)
> Amazon DynamoDB is a fast and flexible NoSQL database service for all
> applications that need consistent, single-digit millisecond latency at any
> scale. It is a fully managed database and supports both document and
> key-value data models. Its flexible data model and reliable performance make
> it a great fit for mobile, web, gaming, ad-tech, IoT, and many other
> applications. -- [AWS DynamoDB Homepage](http://aws.amazon.com/dynamodb/)
JanusGraph + DynamoDB = Distributed Graph Database - Cluster Host Management
[](https://travis-ci.org/awslabs/dynamodb-janusgraph-storage-backend)
## Features
The following is a list of features of the Amazon DynamoDB Storage Backend for
JanusGraph.
* AWS managed authentication and authorization.
* Configure table prefix to allow multiple graphs to be stored in a single
account in the same region.
* Full graph traversals with rate limited table scans.
* Flexible data model allows configuration between single-item and
multiple-item model based on graph size and utilization.
* Test graph locally with DynamoDB Local.
* Integrated with JanusGraph metrics.
* JanusGraph 0.2.0 and TinkerPop 3.2.6 compatibility.
* Upgrade compatibility from Titan 1.0.0.
## Getting Started
This example populates a JanusGraph database backed by DynamoDB Local using
the
[Marvel Universe Social Graph](https://aws.amazon.com/datasets/5621954952932508).
The graph has a vertex per comic book character with an edge to each of the
comic books in which they appeared.
### Load a subset of the Marvel Universe Social Graph
1. Install the prerequisites (Git, JDK 1.8, Maven, Docker, wget, gpg) of this tutorial.
The command below uses a
[convenience script for Amazon Linux](https://raw.githubusercontent.com/awslabs/dynamodb-janusgraph-storage-backend/master/src/test/resources/install-reqs.sh)
on EC2 instances to install Git, Open JDK 1.8, Maven, Docker and Docker Compose.
It adds the ec2-user to the docker group so that you can
[execute Docker commands without using sudo](http://docs.aws.amazon.com/AmazonECS/latest/developerguide/docker-basics.html).
Log out and back in to effect changes on ec2-user.
```bash
curl https://raw.githubusercontent.com/awslabs/dynamodb-janusgraph-storage-backend/master/src/test/resources/install-reqs.sh | bash
exit
```
2. Clone the repository and change directories.
```bash
git clone https://github.com/awslabs/dynamodb-janusgraph-storage-backend.git && cd dynamodb-janusgraph-storage-backend
```
3. Use Docker and Docker Compose to bake DynamoDB Local into a container and start Gremlin Server with the DynamoDB
Storage Backend for JanusGraph installed.
```bash
docker build -t awslabs/dynamodblocal ./src/test/resources/dynamodb-local-docker \
&& src/test/resources/install-gremlin-server.sh \
&& cp server/dynamodb-janusgraph-storage-backend-*.zip src/test/resources/dynamodb-janusgraph-docker \
&& mvn docker:build -Pdynamodb-janusgraph-docker \
&& docker-compose -f src/test/resources/docker-compose.yml up -d \
&& docker exec -i -t dynamodb-janusgraph /var/jg/bin/gremlin.sh
```
4. After the Gremlin shell starts, set it up to execute commands remotely.
```groovy
:remote connect tinkerpop.server conf/remote.yaml session
:remote console
```
5. Load the first 100 lines of the Marvel graph using the Gremlin shell.
```groovy
com.amazon.janusgraph.example.MarvelGraphFactory.load(graph, 100, false)
```
6. Print the characters and the comic-books they appeared in where the
characters had a weapon that was a shield or claws.
```groovy
g.V().has('weapon', within('shield','claws')).as('weapon', 'character', 'book').select('weapon', 'character','book').by('weapon').by('character').by(__.out('appeared').values('comic-book'))
```
7. Print the characters and the comic-books they appeared in where the
characters had a weapon that was not a shield or claws.
```groovy
g.V().has('weapon').has('weapon', without('shield','claws')).as('weapon', 'character', 'book').select('weapon', 'character','book').by('weapon').by('character').by(__.out('appeared').values('comic-book'))
```
8. Print a sorted list of the characters that appear in comic-book AVF 4.
```groovy
g.V().has('comic-book', 'AVF 4').in('appeared').values('character').order()
```
9. Print a sorted list of the characters that appear in comic-book AVF 4 that
have a weapon that is not a shield or claws.
```groovy
g.V().has('comic-book', 'AVF 4').in('appeared').has('weapon', without('shield','claws')).values('character').order()
```
10. Exit remote mode and Control-C to quit.
```groovy
:remote console
```
11. Clean up the composed Docker containers.
```bash
docker-compose -f src/test/resources/docker-compose.yml stop
```
### Load the Graph of the Gods
1. Repeat steps 3 and 4 of the Marvel graph section, cleaning up the server directory beforehand with `rm -rf server`.
2. Load the Graph of the Gods.
```groovy
GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true)
```
3. Now you can follow the rest of the
[JanusGraph Getting Started](http://docs.janusgraph.org/0.1.0/getting-started.html#_global_graph_indices)
documentation, starting from the Global Graph Indeces section. See the
`scriptEngines/gremlin-groovy/scripts` list element in the Gremlin Server YAML
file for more information about what is in scope in the remote environment.
4. Alternatively, repeat steps 1 through 8 of the Marvel graph section and
follow the examples in the
[TinkerPop documentation](http://tinkerpop.apache.org/docs/3.2.3/reference/#_mutating_the_graph).
Skip the `TinkerGraph.open()` step as the remote execution environment already has a
`graph` variable set up. TinkerPop have
[other tutorials](http://tinkerpop.apache.org/docs/3.2.3/#tutorials) available as well.
### Run Gremlin on Gremlin Server in EC2 using CloudFormation templates
The DynamoDB Storage Backend for JanusGraph includes CloudFormation templates that
creates a VPC, an EC2 instance in the VPC, installs Gremlin Server with the
DynamoDB Storage Backend for JanusGraph installed, and starts the Gremlin Server
Websocket endpoint. Also included are templates that create the graph's DynamoDB tables.
The Network ACL of the VPC includes just enough access to allow:
- you to connect to the instance using SSH and create tunnels (SSH inbound)
- the EC2 instance to download yum updates from central repositories (HTTP
outbound)
- the EC2 instance to download your dynamodb.properties file and the Gremlin Server
package from S3 (HTTPS outbound)
- the EC2 instance to connect to DynamoDB (HTTPS outbound)
- the ephemeral ports required to support the data flow above, in each
direction
Requirements for running this CloudFormation template include two items.
- You require an SSH key for EC2 instances must exist in the region you plan to
create the Gremlin Server stack.
- You require permission to call the ec2:DescribeKeyPairs API when creating a stack
from the AWS console.
- You need to have created an IAM role in the region that has S3 Read access
and DynamoDB full access, the very minimum policies required to run this
CloudFormation stack. S3 read access is required to provide the dynamodb.properties
file to the stack in cloud-init. DynamoDB full access is required because the
DynamoDB Storage Backend for JanusGraph can create and delete tables, and read and
write data in those tables.
Note, this cloud formation template downloads
the JanusGraph zip files available on the
[JanusGraph downloads page](https://github.com/JanusGraph/janusgraph/releases).
The CloudFormation template downloads these packages and builds and adds the
DynamoDB Storage Backend for JanusGraph with its dependencies.
#### CloudFormation Template table
Below you can find a list of CloudFormation templates discussed in this document,
and links to launch each stack in CloudFormation and to view the stack in the designer.
| Template name | Description | View |
|----------------------------|------------------------------------------------------------|------|
| Single-Item Model Tables | Set up six graph tables with the single item data model. | [View](https://raw.githubusercontent.com/awslabs/dynamodb-titan-storage-backend/master/dynamodb-janusgraph-tables-single.yaml) |
| Multiple-Item Model Tables | Set up six graph tables with the multiple item data model. | [View](https://raw.githubusercontent.com/awslabs/dynamodb-titan-storage-backend/master/dynamodb-janusgraph-tables-multiple.yaml) |
| Gremlin Server on DynamoDB | The HTTP user agent header to send with all requests. | [View](https://raw.githubusercontent.com/awslabs/dynamodb-titan-storage-backend/master/dynamodb-janusgraph-storage-backend-cfn.yaml) |
#### Instructions to Launch CloudFormation Stacks
1. Choose between the single and multiple item data models and create your graph tables
with the corresponding CloudFormation template above by downloading it and passing it
to the CloudFormation console. Note, the configuration provided in
`src/test/resources/dynamodb.properties` assumes that you
will deploy the stack in us-west-2 and that you will use the multiple item model.
2. Inspect the latest version of the Gremlin Server on DynamoDB stack in the third row above.
3. Download the template from the third row to your computer and use it to create the Gremlin
Server on DynamoDB stack.
4. On the Select Template page, name your Gremlin Server stack and select the
CloudFormation template that you just downloaded.
5. On the Specify Parameters page, you need to specify the following:
* EC2 Instance Type
* The Gremlin Server port, default 8182.
* The S3 URL to your dynamodb.properties configuration file
* The name of your pre-existing EC2 SSH key. Be sure to `chmod 400` on your key as
EC2 instance will reject connections if permissions on the key are
[too open](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/TroubleshootingInstancesConnecting.html#w1ab1c28c17c21).
* The network whitelist for the SSH protocol. You will need to allow incoming
connections via SSH to enable the SSH tunnels that will secure Websockets connections
to Gremlin Server.
* The path to an IAM role that has the minimum amount of privileges to run this
CloudFormation script and run Gremlin Server with the DynamoDB Storage Backend for
JanusGraph. This role will require S3 read to get the dynamodb.properties file, and DynamoDB full
access to create tables and read and write items in those tables. This IAM role needs to be created with
a STS trust relationship including `ec2.amazonaws.com` as an identity provider. The easiest way to do
this is to [create a new role on the IAM console](https://console.aws.amazon.com/iam/home?region=us-west-2#/roles)
and from the AWS Service Role list in the accordion, select Amazon EC2, and add the AmazonDynamoDBFullAccess
and AmazonS3ReadOnlyAccess managed policies.
6. On the Options page, click Next.
7. On the Review page, select "I acknowledge that this template might cause AWS
CloudFormation to create IAM resources." Then, click Create.
8. Start the Gremlin console on the host through SSH. You can just copy paste the `GremlinShell` output of the
CloudFormation template and run it on your command line.
9. Repeat steps 4 and onwards of the Marvel graph section above.
## Data Model
The Amazon DynamoDB Storage Backend for JanusGraph has a flexible data model that
allows clients to select the data model for each JanusGraph backend table. Clients
can configure tables to use either a single-item model or a multiple-item model.
### Single-Item Model
The single-item model uses a single DynamoDB item to store all values for a
single key. In terms of JanusGraph backend implementations, the key becomes the
DynamoDB hash key, and each column becomes an attribute name and the column
value is stored in the respective attribute value.
This is definitely the most efficient implementation, but beware of the 400kb
limit DynamoDB imposes on items. It is best to only use this on tables you are
sure will not surpass the item size limit. Graphs with low vertex degree and
low number of items per index can take advantage of this implementation.
### Multiple-Item Model
The multiple-item model uses multiple DynamoDB items to store all values for a
single key. In terms of JanusGraph backend implementations, the key becomes the
DynamoDB hash key, and each column becomes the range key in its own item.
The column values are stored in its own attribute.
The multiple item model is less efficient than the single-item during initial
graph loads, but it gets around the 400kb limitation. The multiple-item model
uses range Query calls instead of GetItem calls to get the necessary column
values.
## DynamoDB Specific Configuration
Each configuration option has a certain mutability level that governs whether
and how it can be modified after the database is opened for the first time. The
following listing describes the mutability levels.
1. **FIXED** - Once the database has been opened, these configuration options
cannot be changed for the entire life of the database
2. **GLOBAL_OFFLINE** - These options can only be changed for the entire
database cluster at once when all instances are shut down
3. **GLOBAL** - These options can only be changed globally across the entire
database cluster
4. **MASKABLE** - These options are global but can be overwritten by a local
configuration file
5. **LOCAL** - These options can only be provided through a local configuration
file
Leading namespace names are shortened and sometimes spaces were inserted in long
strings to make sure the tables below are formatted correctly.
### General DynamoDB Configuration Parameters
All of the following parameters are in the `storage` (`s`) namespace, and most
are in the `storage.dynamodb` (`s.d`) namespace subset.
| Name | Description | Datatype | Default Value | Mutability |
|-----------------|-------------|----------|---------------|------------|
| `s.backend` | The primary persistence provider used by JanusGraph. To use DynamoDB you must set this to `com.amazon.janusgraph.diskstorage. dynamodb.DynamoDBStoreManager` | String | | LOCAL |
| `s.d.prefix` | A prefix to put before the JanusGraph table name. This allows clients to have multiple graphs in the same AWS DynamoDB account in the same region. | String | jg | LOCAL |
| `s.d.metrics-prefix` | Prefix on the codahale metric names emitted by DynamoDBDelegate. | String | d | LOCAL |
| `s.d.force-consistent-read` | This feature sets the force consistent read property on DynamoDB calls. | Boolean | true | LOCAL |
| `s.d.enable-parallel-scan` | This feature changes the scan behavior from a sequential scan (with consistent key order) to a segmented, parallel scan. Enabling this feature will make full graph scans faster, but it may cause this backend to be incompatible with Titan's OLAP library. | Boolean | false | LOCAL |
| `s.d.max-self-throttled-retries` | The number of retries that the backend should attempt and self-throttle. | Integer | 60 | LOCAL |
| `s.d.initial-retry-millis` | The amount of time to initially wait (in milliseconds) when retrying self-throttled DynamoDB API calls. | Integer | 25 | LOCAL |
| `s.d.control-plane-rate` | The rate in permits per second at which to issue DynamoDB control plane requests (CreateTable, UpdateTable, DeleteTable, ListTables, DescribeTable). | Double | 10 | LOCAL |
| `s.d.native-locking` | Set this to false if you need to use JanusGraph's locking mechanism for remote lock expiry. | Boolean | true | LOCAL |
| `s.d.use-titan-ids` | Set this to true if you are migrating from Titan to JanusGraph so that you do not have to copy your titan_ids table. | Boolean | false | LOCAL |
### DynamoDB KeyColumnValue Store Configuration Parameters
Some configurations require specifications for each of the JanusGraph backend
Key-Column-Value stores. Here is a list of the default JanusGraph backend
Key-Column-Value stores:
* edgestore
* graphindex
* janusgraph_ids (this used to be called titan_ids in Titan)
* system_properties
* systemlog
* txlog
Any store you define in the umbrella `storage.dynamodb.stores.*` namespace that starts
with `ulog_` will be used for user-defined transaction logs.
Again, if you opt out of storage-native locking with the
`storage.dynamodb.native-locking = false` configuration, you will need to configure the
data model, initial capacity and rate limiters for the three following stores:
* edgestore_lock_
* graphindex_lock_
* system_properties_lock_
You can configure the initial read and write capacity, rate limits, scan limits and
data model for each KCV graph store. You can always scale up and down the read and
write capacity of your tables in the DynamoDB console. If you have a write once,
read many workload, or you are running a bulk data load, it is useful to adjust the
capacity of edgestore and graphindex tables as necessary in the DynamoDB console,
and decreasing the allocated capacity and rate limiters afterwards.
For details about these Key-Column-Value stores, please see
[Store Mapping](https://github.com/BillBaird/delftswa-aurelius-titan/blob/master/SA-doc/Mapping.md)
and
[JanusGraph Data Model](http://docs.janusgraph.org/0.1.0/data-model.html).
All of these configuration parameters are in the `storage.dynamodb.stores`
(`s.d.s`) umbrella namespace subset. In the tables below these configurations
have the text `t` where the JanusGraph store name should go.
When upgrading from Titan 1.0.0, you will need to set the
[ids.store-name configuration](http://docs.janusgraph.org/latest/config-ref.html#_ids)
to `titan_ids` to avoid re-using id ranges that are already assigned.
| Name | Description | Datatype | Default Value | Mutability |
|-----------------|-------------|----------|---------------|------------|
| `s.d.s.t.data-model` | SINGLE means that all the values for a given key are put into a single DynamoDB item. A SINGLE is efficient because all the updates for a single key can be done atomically. However, the tradeoff is that DynamoDB has a 400k limit per item so it cannot hold much data. MULTI means that each 'column' is used as a range key in DynamoDB so a key can span multiple items. A MULTI implementation is slightly less efficient than SINGLE because it must use DynamoDB Query rather than a direct lookup. It is HIGHLY recommended to use MULTI for edgestore and graphindex unless your graph has very low max degree.| String | MULTI | FIXED |
| `s.d.s.t.initial-capacity-read` | Define the initial read capacity for a given DynamoDB table. Make sure to replace the `s` with your actual table name. | Integer | 4 | LOCAL |
| `s.d.s.t.initial-capacity-write` | Define the initial write capacity for a given DynamoDB table. Make sure to replace the `s` with your actual table name. | Integer | 4 | LOCAL |
| `s.d.s.t.read-rate` | The max number of reads per second. | Double | 4 | LOCAL |
| `s.d.s.t.write-rate` | Used to throttle write rate of given table. The max number of writes per second. | Double | 4 | LOCAL |
| `s.d.s.t.scan-limit` | The maximum number of items to evaluate (not necessarily the number of matching items). If DynamoDB processes the number of items up to the limit while processing the results, it stops the operation and returns the matching values up to that point, and a key in LastEvaluatedKey to apply in a subsequent operation, so that you can pick up where you left off. Also, if the processed data set size exceeds 1 MB before DynamoDB reaches this limit, it stops the operation and returns the matching values up to the limit, and a key in LastEvaluatedKey to apply in a subsequent operation to continue the operation. | Integer | 10000 | LOCAL |
### DynamoDB Client Configuration Parameters
All of these configuration parameters are in the `storage.dynamodb.client`
(`s.d.c`) namespace subset, and are related to the DynamoDB SDK client
configuration.
| Name | Description | Datatype | Default Value | Mutability |
|-----------------|-------------|----------|---------------|------------|
| `s.d.c.connection-timeout` | The amount of time to wait (in milliseconds) when initially establishing a connection before giving up and timing out. | Integer | 60000 | LOCAL |
| `s.d.c.connection-ttl` | The expiration time (in milliseconds) for a connection in the connection pool. | Integer | 60000 | LOCAL |
| `s.d.c.connection-max` | The maximum number of allowed open HTTP connections.| Integer | 10 | LOCAL |
| `s.d.c.retry-error-max` | The maximum number of retry attempts for failed retryable requests (ex: 5xx error responses from services).| Integer | 0 | LOCAL |
| `s.d.c.use-gzip` | Sets whether gzip compression should be used. | Boolean | false | LOCAL |
| `s.d.c.use-reaper` | Sets whether the IdleConnectionReaper is to be started as a daemon thread. | Boolean | true | LOCAL |
| `s.d.c.user-agent` | The HTTP user agent header to send with all requests.| String | | LOCAL |
| `s.d.c.endpoint` | Sets the service endpoint to use for connecting to DynamoDB. | String | | LOCAL |
| `s.d.c.signing-region` | Sets the signing region to use for signing requests to DynamoDB. Required. | String | | LOCAL |
#### DynamoDB Client Proxy Configuration Parameters
All of these configuration parameters are in the
`storage.dynamodb.client.proxy` (`s.d.c.p`) namespace subset, and are related to
the DynamoDB SDK client proxy configuration.
| Name | Description | Datatype | Default Value | Mutability |
|-----------------|-------------|----------|---------------|------------|
| `s.d.c.p.domain` | The optional Windows domain name for configuration an NTLM proxy.| String | | LOCAL |
| `s.d.c.p.workstation` | The optional Windows workstation name for configuring NTLM proxy support.| String | | LOCAL |
| `s.d.c.p.host` | The optional proxy host the client will connect through.| String | | LOCAL |
| `s.d.c.p.port` | The optional proxy port the client will connect through.| String | | LOCAL |
| `s.d.c.p.username` | The optional proxy user name to use if connecting through a proxy.| String | | LOCAL |
| `s.d.c.p.password` | The optional proxy password to use when connecting through a proxy.| String | | LOCAL |
#### DynamoDB Client Socket Configuration Parameters
All of these configuration parameters are in the `storage.dynamodb.client.socket`
(`s.d.c.s`) namespace subset, and are related to the DynamoDB SDK client socket
configuration.
| Name | Description | Datatype | Default Value | Mutability |
|-----------------|-------------|----------|---------------|------------|
| `s.d.c.s.buffer-send-hint` | The optional size hints (in bytes) for the low level TCP send and receive buffers.| Integer | 1048576 | LOCAL |
| `s.d.c.s.buffer-recv-hint` | The optional size hints (in bytes) for the low level TCP send and receive buffers.| Integer | 1048576 | LOCAL |
| `s.d.c.s.timeout` | The amount of time to wait (in milliseconds) for data to be transfered over an established, open connection before the connection times out and is closed.| Long | 50000 | LOCAL |
| `s.d.c.s.tcp-keep-alive` | Sets whether or not to enable TCP KeepAlive support at the socket level. Not used at the moment. | Boolean | | LOCAL |
#### DynamoDB Client Executor Configuration Parameters
All of these configuration parameters are in the `storage.dynamodb.client.executor`
(`s.d.c.e`) namespace subset, and are related to the DynamoDB SDK client
executor / thread-pool configuration.
| Name | Description | Datatype | Default Value | Mutability |
|-----------------|-------------|----------|---------------|------------|
| `s.d.c.e.core-pool-size` | The core number of threads for the DynamoDB async client. | Integer | 25| LOCAL |
| `s.d.c.e.max-pool-size` | The maximum allowed number of threads for the DynamoDB async client. | Integer | 50 | LOCAL |
| `s.d.c.e.keep-alive` | The time limit for which threads may remain idle before being terminated for the DynamoDB async client. | Integer | | LOCAL |
| `s.d.c.e.max-queue-length` | The maximum size of the executor queue before requests start getting run in the caller. | Integer | 1024 | LOCAL |
| `s.d.c.e.max-concurrent-operations` | The expected number of threads expected to be using a single JanusGraph instance. Used to allocate threads to batch operations. | Integer | 1 | LOCAL |
#### DynamoDB Client Credential Configuration Parameters
All of these configuration parameters are in the `storage.dynamodb.client.credentials`
(`s.d.c.c`) namespace subset, and are related to the DynamoDB SDK client
credential configuration.
| Name | Description | Datatype | Default Value | Mutability |
|-----------------|-------------|----------|---------------|------------|
| `s.d.c.c.class-name` | Specify the fully qualified class that implements AWSCredentialsProvider or AWSCredentials. | String | `com.amazonaws.auth. BasicAWSCredentials` | LOCAL |
| `s.d.c.c.constructor-args` | Comma separated list of strings to pass to the credentials constructor. | String | `accessKey,secretKey` | LOCAL |
## Upgrading from Titan 1.0.0
Earlier versions of this software supported Titan 1.0.0. This software supports upgrading from
the DynamoDB Storage Backend for Titan 1.0.0 by following the steps to update your configuration below.
1. Set the JanusGraph configuration option
[`ids.store-name=titan_ids`](http://docs.janusgraph.org/latest/config-ref.html#_ids). This allows you to reuse your
`titan_ids` table.
2. Update the classpath to the DynamoDB Storage Backend to use the latest package name,
`storage.backend=com.amazon.janusgraph.diskstorage.dynamodb.DynamoDBStoreManager` .
## Run all tests against DynamoDB Local on an EC2 Amazon Linux AMI
1. Install dependencies. For Amazon Linux:
```bash
sudo wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo \
-O /etc/yum.repos.d/epel-apache-maven.repo
sudo sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo
sudo yum update -y && sudo yum upgrade -y
sudo yum install -y apache-maven sqlite-devel git java-1.8.0-openjdk-devel
sudo alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java
sudo alternatives --set javac /usr/lib/jvm/java-1.8.0-openjdk.x86_64/bin/javac
git clone https://github.com/awslabs/dynamodb-janusgraph-storage-backend.git
cd dynamodb-janusgraph-storage-backend && mvn install
```
2. Open a screen so that you can log out of the EC2 instance while running tests with `screen`.
3. Run the single-item data model tests.
```bash
mvn verify -P integration-tests \
-Dexclude.category=com.amazon.janusgraph.testcategory.MultipleItemTestCategory \
-Dinclude.category="**/*.java" > o 2>&1
```
4. Run the multiple-item data model tests.
```bash
mvn verify -P integration-tests \
-Dexclude.category=com.amazon.janusgraph.testcategory.SingleItemTestCategory \
-Dinclude.category="**/*.java" > o 2>&1
```
5. Run other miscellaneous tests.
```bash
mvn verify -P integration-tests -Dinclude.category="**/*.java" \
-Dgroups=com.amazon.janusgraph.testcategory.IsolateRemainingTestsCategory > o 2>&1
```
6. Exit the screen with `CTRL-A D` and logout of the EC2 instance.
7. Monitor the CPU usage of your EC2 instance in the EC2 console. The single-item tests
may take at least 1 hour and the multiple-item tests may take at least 2 hours to run.
When CPU usage goes to zero, that means the tests are done.
8. Log back into the EC2 instance and resume the screen with `screen -r` to
review the test results.
```bash
cd target/surefire-reports && grep testcase *.xml | grep -v "\/"
```
9. Terminate the instance when done.
================================================
FILE: buildspec.yml
================================================
version: 0.1
phases:
build:
commands:
- echo Build started on `date`
- mvn install
# From README.md, use: MultipleItemTestCategory, SingleItemTestCategory, IsolateGraphFailingTestCategory for CATEGORY
- mvn verify -Pintegration-tests -Dgroups="com.amazon.janusgraph.testcategory.${CATEGORY}" -Dinclude.category="**/*.java"
- aws cloudformation validate-template --template-body `pwd | sed -e 's/\//\/\//g' -e 's/^/file:\//' -e 's/$/\/\/dynamodb-janusgraph-storage-backend-cfn\.yaml/'`
- src/test/resources/install-gremlin-server.sh
post_build:
commands:
- echo Build completed on `date`
artifacts:
files:
- target/**/*
================================================
FILE: checkstyle.xml
================================================
================================================
FILE: dynamodb-janusgraph-storage-backend-cfn.yaml
================================================
---
Description: This stack creates a VPC, an EC2 Amazon Linux host in the VPC with a
Public IP, and deploys Gremlin Server on it. **WARNING** This template creates
an Amazon EC2 instance. You will be billed for the AWS resources used if you
create a stack from this template. Amazon Linux created on 2018-06-22.
AWSTemplateFormatVersion: '2010-09-09'
Mappings:
AWSRegionArch2AMI:
ap-northeast-1:
HVM64: ami-449f483b
HVMG2: ami-9c9443e3
PV64: ami-6593441a
ap-northeast-2:
HVM64: ami-5bd46135
HVMG2: ami-ebc47185
ap-south-1:
HVM64: ami-bc83a9d3
HVMG2: ami-5a8da735
ap-southeast-1:
HVM64: ami-d6fdfeaa
HVMG2: ami-ed838091
PV64: ami-c8fcffb4
ap-southeast-2:
HVM64: ami-4ff8212d
HVMG2: ami-33f92051
PV64: ami-c4c71ea6
ca-central-1:
HVM64: ami-49e86a2d
HVMG2: ami-03e86a67
eu-central-1:
HVM64: ami-e056690b
HVMG2: ami-a058674b
PV64: ami-1a744bf1
eu-west-1:
HVM64: ami-41505fab
HVMG2: ami-e4515e0e
PV64: ami-3c5758d6
eu-west-2:
HVM64: ami-e2b35a85
HVMG2: ami-b2b55cd5
eu-west-3:
HVM64: ami-78f24205
HVMG2: ami-d50bbaa8
sa-east-1:
HVM64: ami-09d58f65
HVMG2: ami-83d58fef
PV64: ami-6dd58f01
us-east-1:
HVM64: ami-f316478c
HVMG2: ami-cfe4b2b0
PV64: ami-b41445cb
us-east-2:
HVM64: ami-ae0f36cb
HVMG2: ami-40142d25
us-west-1:
HVM64: ami-25bf5946
HVMG2: ami-0e86606d
PV64: ami-1a876179
us-west-2:
HVM64: ami-39d39d41
HVMG2: ami-0ad99772
PV64: ami-21d09e59
AWSInstanceType2Arch:
t2.nano:
Arch: HVM64
t2.micro:
Arch: HVM64
t2.small:
Arch: HVM64
t2.medium:
Arch: HVM64
t2.large:
Arch: HVM64
t2.xlarge:
Arch: HVM64
t2.2xlarge:
Arch: HVM64
m4.large:
Arch: HVM64
m4.xlarge:
Arch: HVM64
m4.2xlarge:
Arch: HVM64
m4.4xlarge:
Arch: HVM64
m4.10xlarge:
Arch: HVM64
m4.16xlarge:
Arch: HVM64
m5.large:
Arch: HVM64
m5.xlarge:
Arch: HVM64
m5.2xlarge:
Arch: HVM64
m5.4xlarge:
Arch: HVM64
m5.12xlarge:
Arch: HVM64
m5.24xlarge:
Arch: HVM64
m5d.large:
Arch: HVM64
m5d.xlarge:
Arch: HVM64
m5d.2xlarge:
Arch: HVM64
m5d.4xlarge:
Arch: HVM64
m5d.12xlarge:
Arch: HVM64
m5d.24xlarge:
Arch: HVM64
c4.large:
Arch: HVM64
c4.xlarge:
Arch: HVM64
c4.2xlarge:
Arch: HVM64
c4.4xlarge:
Arch: HVM64
c4.8xlarge:
Arch: HVM64
c5.large:
Arch: HVM64
c5.xlarge:
Arch: HVM64
c5.2xlarge:
Arch: HVM64
c5.4xlarge:
Arch: HVM64
c5.9xlarge:
Arch: HVM64
c5.18xlarge:
Arch: HVM64
c5d.xlarge:
Arch: HVM64
c5d.2xlarge:
Arch: HVM64
c5d.4xlarge:
Arch: HVM64
c5d.9xlarge:
Arch: HVM64
c5d.18xlarge:
Arch: HVM64
r4.large:
Arch: HVM64
r4.xlarge:
Arch: HVM64
r4.2xlarge:
Arch: HVM64
r4.4xlarge:
Arch: HVM64
r4.8xlarge:
Arch: HVM64
r4.16xlarge:
Arch: HVM64
x1.16xlarge:
Arch: HVM64
x1.32xlarge:
Arch: HVM64
x1e.xlarge:
Arch: HVM64
x1e.2xlarge:
Arch: HVM64
x1e.4xlarge:
Arch: HVM64
x1e.8xlarge:
Arch: HVM64
x1e.16xlarge:
Arch: HVM64
x1e.32xlarge:
Arch: HVM64
d2.xlarge:
Arch: HVM64
d2.2xlarge:
Arch: HVM64
d2.4xlarge:
Arch: HVM64
d2.8xlarge:
Arch: HVM64
h1.2xlarge:
Arch: HVM64
h1.4xlarge:
Arch: HVM64
h1.8xlarge:
Arch: HVM64
h1.16xlarge:
Arch: HVM64
i3.large:
Arch: HVM64
i3.xlarge:
Arch: HVM64
i3.2xlarge:
Arch: HVM64
i3.4xlarge:
Arch: HVM64
i3.8xlarge:
Arch: HVM64
i3.16xlarge:
Arch: HVM64
i3.metal:
Arch: HVM64
f1.2xlarge:
Arch: HVM64
f1.16xlarge:
Arch: HVM64
g3.4xlarge:
Arch: HVM64
g3.8xlarge:
Arch: HVM64
g3.16xlarge:
Arch: HVM64
p2.xlarge:
Arch: HVM64
p2.8xlarge:
Arch: HVM64
p2.16xlarge:
Arch: HVM64
p3.2xlarge:
Arch: HVM64
p3.8xlarge:
Arch: HVM64
p3.16xlarge:
Arch: HVM64
Parameters:
InstanceType:
Description: EC2 instance type
Type: String
Default: m4.10xlarge
AllowedValues:
- t2.nano
- t2.micro
- t2.small
- t2.medium
- t2.large
- t2.xlarge
- t2.2xlarge
- m4.large
- m4.xlarge
- m4.2xlarge
- m4.4xlarge
- m4.10xlarge
- m4.16xlarge
- m5.large
- m5.xlarge
- m5.2xlarge
- m5.4xlarge
- m5.12xlarge
- m5.24xlarge
- m5d.large
- m5d.xlarge
- m5d.2xlarge
- m5d.4xlarge
- m5d.12xlarge
- m5d.24xlarge
- c4.large
- c4.xlarge
- c4.2xlarge
- c4.4xlarge
- c4.8xlarge
- c5.large
- c5.xlarge
- c5.2xlarge
- c5.4xlarge
- c5.9xlarge
- c5.18xlarge
- c5d.xlarge
- c5d.2xlarge
- c5d.4xlarge
- c5d.9xlarge
- c5d.18xlarge
- r4.large
- r4.xlarge
- r4.2xlarge
- r4.4xlarge
- r4.8xlarge
- r4.16xlarge
- x1.16xlarge
- x1.32xlarge
- x1e.xlarge
- x1e.2xlarge
- x1e.4xlarge
- x1e.8xlarge
- x1e.16xlarge
- x1e.32xlarge
- d2.xlarge
- d2.2xlarge
- d2.4xlarge
- d2.8xlarge
- h1.2xlarge
- h1.4xlarge
- h1.8xlarge
- h1.16xlarge
- i3.large
- i3.xlarge
- i3.2xlarge
- i3.4xlarge
- i3.8xlarge
- i3.16xlarge
- i3.metal
- f1.2xlarge
- f1.16xlarge
- g3.4xlarge
- g3.8xlarge
- g3.16xlarge
- p2.xlarge
- p2.8xlarge
- p2.16xlarge
- p3.2xlarge
- p3.8xlarge
- p3.16xlarge
ConstraintDescription: Must be a valid EC2 instance type.
KeyName:
Description: Name of existing EC2 SSH key
Type: AWS::EC2::KeyPair::KeyName
Default: ''
MinLength: '0'
MaxLength: '255'
AllowedPattern: "[\\x20-\\x7E]*"
ConstraintDescription: can contain only ASCII characters.
SSHLocation:
Description: CIDR range allowed to SSH into EC2 instance
Type: String
MinLength: '9'
MaxLength: '18'
Default: 0.0.0.0/32
AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
VPCRange:
Description: CIDR range to assign to new VPC
Type: String
MinLength: '9'
MaxLength: '18'
Default: 10.0.0.0/16
AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
SubnetRange:
Description: CIDR range to assign to new subnet in VPC
Type: String
MinLength: '9'
MaxLength: '18'
Default: 10.0.0.0/24
AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
GremlinServerPort:
Description: The port to serve Gremlin Server Websockets API out of
Type: String
MinLength: '1'
MaxLength: '5'
Default: '8182'
AllowedPattern: "(\\d{1,5})"
ConstraintDescription: must be a valid port.
StorageBackendPropertiesFileS3Url:
Description: S3 URL to the JanusGraph configuration file
Type: String
MinLength: '1'
JanusGraphInstanceProfilePath:
Description: The IAM path
Type: String
Default: "/"
MinLength: '1'
JanusGraphInstanceProfileRole:
Description: An IAM role that includes at least S3 read and DynamoDB full access
Type: String
MinLength: '1'
Resources:
InstanceProfile:
Type: AWS::IAM::InstanceProfile
Properties:
Path:
Ref: JanusGraphInstanceProfilePath
Roles:
- Ref: JanusGraphInstanceProfileRole
VPC:
Type: AWS::EC2::VPC
Properties:
CidrBlock:
Ref: VPCRange
EnableDnsSupport: 'true'
EnableDnsHostnames: 'true'
Tags:
- Key: Application
Value:
Ref: AWS::StackId
Subnet:
Type: AWS::EC2::Subnet
Properties:
VpcId:
Ref: VPC
CidrBlock:
Ref: SubnetRange
Tags:
- Key: Application
Value:
Ref: AWS::StackId
InternetGateway:
Type: AWS::EC2::InternetGateway
Properties:
Tags:
- Key: Application
Value:
Ref: AWS::StackId
AttachGateway:
Type: AWS::EC2::VPCGatewayAttachment
Properties:
VpcId:
Ref: VPC
InternetGatewayId:
Ref: InternetGateway
RouteTable:
Type: AWS::EC2::RouteTable
Properties:
VpcId:
Ref: VPC
Tags:
- Key: Application
Value:
Ref: AWS::StackId
Route:
Type: AWS::EC2::Route
DependsOn: AttachGateway
Properties:
RouteTableId:
Ref: RouteTable
DestinationCidrBlock: 0.0.0.0/0
GatewayId:
Ref: InternetGateway
SubnetRouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
SubnetId:
Ref: Subnet
RouteTableId:
Ref: RouteTable
NetworkAcl:
Type: AWS::EC2::NetworkAcl
Properties:
VpcId:
Ref: VPC
Tags:
- Key: Application
Value:
Ref: AWS::StackId
InboundSSHNetworkAclEntry:
Type: AWS::EC2::NetworkAclEntry
Properties:
NetworkAclId:
Ref: NetworkAcl
RuleNumber: '101'
Protocol: '6'
RuleAction: allow
Egress: 'false'
CidrBlock:
Ref: SSHLocation
PortRange:
From: '22'
To: '22'
InboundResponsePortsNetworkAclEntry:
Type: AWS::EC2::NetworkAclEntry
Properties:
NetworkAclId:
Ref: NetworkAcl
RuleNumber: '102'
Protocol: '6'
RuleAction: allow
Egress: 'false'
CidrBlock: 0.0.0.0/0
PortRange:
From: '1024'
To: '65535'
OutboundHTTPNetworkAclEntry:
Type: AWS::EC2::NetworkAclEntry
Properties:
NetworkAclId:
Ref: NetworkAcl
RuleNumber: '101'
Protocol: '6'
RuleAction: allow
Egress: 'true'
CidrBlock: 0.0.0.0/0
PortRange:
From: '80'
To: '80'
OutboundHTTPSNetworkAclEntry:
Type: AWS::EC2::NetworkAclEntry
Properties:
NetworkAclId:
Ref: NetworkAcl
RuleNumber: '102'
Protocol: '6'
RuleAction: allow
Egress: 'true'
CidrBlock: 0.0.0.0/0
PortRange:
From: '443'
To: '443'
OutboundResponsePortsNetworkAclEntry:
Type: AWS::EC2::NetworkAclEntry
Properties:
NetworkAclId:
Ref: NetworkAcl
RuleNumber: '103'
Protocol: '6'
RuleAction: allow
Egress: 'true'
CidrBlock: 0.0.0.0/0
PortRange:
From: '1024'
To: '65535'
SubnetNetworkAclAssociation:
Type: AWS::EC2::SubnetNetworkAclAssociation
Properties:
SubnetId:
Ref: Subnet
NetworkAclId:
Ref: NetworkAcl
InstanceSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
VpcId:
Ref: VPC
GroupDescription: Enable SSH access via port 22
SecurityGroupIngress:
- IpProtocol: tcp
FromPort: '22'
ToPort: '22'
CidrIp:
Ref: SSHLocation
WebServerInstance:
Type: AWS::EC2::Instance
DependsOn: AttachGateway
Properties:
DisableApiTermination: 'FALSE'
IamInstanceProfile:
Ref: InstanceProfile
ImageId:
Fn::FindInMap:
- AWSRegionArch2AMI
- Ref: AWS::Region
- Fn::FindInMap:
- AWSInstanceType2Arch
- Ref: InstanceType
- Arch
InstanceType:
Ref: InstanceType
KeyName:
Ref: KeyName
Monitoring: 'false'
NetworkInterfaces:
- GroupSet:
- Ref: InstanceSecurityGroup
AssociatePublicIpAddress: 'true'
DeviceIndex: '0'
DeleteOnTermination: 'true'
SubnetId:
Ref: Subnet
Tags:
- Key: Name
Value: Amazon DynamoDB Storage Backend for JanusGraph on AWS
UserData:
Fn::Base64:
Fn::Join:
- ''
- - "#!/bin/bash\n"
- "export SDKMAN_DIR=/usr/local/sdkman && curl -s https://get.sdkman.io | bash && source /usr/local/sdkman/bin/sdkman-init.sh\n"
- "echo 'export SDKMAN_DIR=/usr/local/sdkman; source /usr/local/sdkman/bin/sdkman-init.sh' > /etc/profile.d/sdkman.sh\n"
- "yum update -y -q -e 0 && yum upgrade -y -q -e 0 && yum install -y java-1.8.0-openjdk-devel > /home/ec2-user/yumupdates.log\n"
- "yum remove -y java-1.7.0-openjdk > /home/ec2-user/yumremovejava7.log\n"
- "sdk install maven < /dev/null && set -x\n"
- "mvn -version > /home/ec2-user/maven-installation-settings.log\n"
- "export GREMLIN_SERVER_USERNAME='ec2-user'\n"
- "export LOG_DIR=/var/log/gremlin-server\n"
- "export SERVER_DIRNAME=dynamodb-janusgraph-storage-backend-1.2.0\n"
- "export SERVER_ZIP=${SERVER_DIRNAME}.zip\n"
- "export PACKAGES_DIR=/usr/local/packages\n"
- "export INSTALL_DIR=${PACKAGES_DIR}/${SERVER_DIRNAME}\n"
- "export REPO_PARENT_DIR=/tmp/shm\n"
- "export REPO_ARCHIVE_DIR=${REPO_PARENT_DIR}/dynamodb-janusgraph-storage-backend-master\n"
- "mkdir -p ${LOG_DIR} ${INSTALL_DIR} /tmp/shm\n"
- "export SERVICE_SCRIPT=${INSTALL_DIR}/bin/gremlin-server-service.sh\n"
- "pushd ${REPO_PARENT_DIR}\n"
- "wget https://github.com/awslabs/dynamodb-janusgraph-storage-backend/archive/master.zip && unzip -q master.zip\n"
- "pushd ${REPO_ARCHIVE_DIR}\n"
- "src/test/resources/install-gremlin-server.sh > /home/ec2-user/gremlin-server-installation.log && popd && popd\n"
- "pushd ${PACKAGES_DIR}\n"
- "mv ${REPO_ARCHIVE_DIR}/server/${SERVER_DIRNAME} . && rm -rf ${REPO_PARENT_DIR}/* && chmod u+x ${SERVICE_SCRIPT} && ln -s ${SERVICE_SCRIPT} /etc/init.d/gremlin-server && chkconfig --add gremlin-server\n"
- "export BACKEND_PROPERTIES=${INSTALL_DIR}/conf/gremlin-server/dynamodb.properties\n"
- "aws s3 cp "
- Ref: "StorageBackendPropertiesFileS3Url"
- " ${BACKEND_PROPERTIES}\n"
- "chown -R ${GREMLIN_SERVER_USERNAME}:${GREMLIN_SERVER_USERNAME} ${LOG_DIR} ${INSTALL_DIR}\n"
- "ln -s ${INSTALL_DIR}/conf /home/ec2-user/conf && chmod a+r /home/ec2-user/conf\n"
- "service gremlin-server start > /home/ec2-user/gremlin-server-start.log\n\n"
Outputs:
SshTunnels:
Value:
Fn::Join:
- ''
- - ssh -o ServerAliveInterval=50 -nNT -L
- Ref: GremlinServerPort
- ":localhost:"
- Ref: GremlinServerPort
- " -i ${HOME}/.ec2/"
- Ref: KeyName
- ".pem ec2-user@"
- Fn::GetAtt:
- WebServerInstance
- PublicDnsName
Description: Use these SSH tunnels to access Gremlin Server.
GremlinShell:
Value:
Fn::Join:
- ''
- - "ssh -o ServerAliveInterval=50 -t -i ${HOME}/.ec2/"
- Ref: KeyName
- ".pem ec2-user@"
- Fn::GetAtt:
- WebServerInstance
- PublicDnsName
- " /usr/local/packages/dynamodb-janusgraph-storage-backend-1.2.0/bin/gremlin.sh"
Description: Use this remote shell to interact with the graph.
GremlinServerEndpoint:
Value:
Fn::Join:
- ''
- - 'http://localhost:'
- Ref: GremlinServerPort
Description: This is the Gremlin Server Websockets endpoint after creating the
SSH tunnel.
SshAccess:
Value:
Fn::Join:
- ''
- - ssh -o ServerAliveInterval=50 -i ${HOME}/.ec2/
- Ref: KeyName
- ".pem ec2-user@"
- Fn::GetAtt:
- WebServerInstance
- PublicDnsName
Description: This is how you gain remote access to the machine.
Note:
Value: Wait while your EC2 host to boot and start Gremlin Server with the Amazon
DynamoDB Storage Backend for JanusGraph.
Description: ''
================================================
FILE: dynamodb-janusgraph-tables-multiple.yaml
================================================
AWSTemplateFormatVersion: "2010-09-09"
Parameters:
TablePrefix:
Description: Table prefix to prepend all table names with
Type: String
Default: jg
MinLength: '1'
AllowedPattern: "[a-zA-Z0-9]+"
EdgestoreReadCapacity:
Description: Read capacity of the edgestore table
Type: Number
Default: 12
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
EdgestoreWriteCapacity:
Description: Write capacity of the edgestore table
Type: Number
Default: 12
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
GraphindexReadCapacity:
Description: Read capacity of the graphindex table
Type: Number
Default: 9
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
GraphindexWriteCapacity:
Description: Write capacity of the graphindex table
Type: Number
Default: 9
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
JanusGraphIdsReadCapacity:
Description: Read capacity of the janusgraph_ids table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
JanusGraphIdsWriteCapacity:
Description: Write capacity of the janusgraph_ids table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemPropertiesReadCapacity:
Description: Read capacity of the system_properties table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemPropertiesWriteCapacity:
Description: Write capacity of the system_properties table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
TxlogReadCapacity:
Description: Read capacity of the txlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
TxlogWriteCapacity:
Description: Write capacity of the txlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemlogReadCapacity:
Description: Read capacity of the systemlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemlogWriteCapacity:
Description: Write capacity of the systemlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
Resources:
Edgestore:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
-
AttributeName: "rk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
-
AttributeName: "rk"
KeyType: "RANGE"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: EdgestoreReadCapacity
WriteCapacityUnits:
Ref: EdgestoreWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _edgestore
Graphindex:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
-
AttributeName: "rk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
-
AttributeName: "rk"
KeyType: "RANGE"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: GraphindexReadCapacity
WriteCapacityUnits:
Ref: GraphindexWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _graphindex
JanusGraphIds:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
-
AttributeName: "rk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
-
AttributeName: "rk"
KeyType: "RANGE"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: JanusGraphIdsReadCapacity
WriteCapacityUnits:
Ref: JanusGraphIdsWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _janusgraph_ids
SystemProperties:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
-
AttributeName: "rk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
-
AttributeName: "rk"
KeyType: "RANGE"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: SystemPropertiesReadCapacity
WriteCapacityUnits:
Ref: SystemPropertiesWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _system_properties
Txlog:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
-
AttributeName: "rk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
-
AttributeName: "rk"
KeyType: "RANGE"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: TxlogReadCapacity
WriteCapacityUnits:
Ref: TxlogWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _txlog
Systemlog:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
-
AttributeName: "rk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
-
AttributeName: "rk"
KeyType: "RANGE"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: SystemlogReadCapacity
WriteCapacityUnits:
Ref: SystemlogWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _systemlog
================================================
FILE: dynamodb-janusgraph-tables-single.yaml
================================================
AWSTemplateFormatVersion: "2010-09-09"
Parameters:
TablePrefix:
Description: Table prefix to prepend all table names with
Type: String
Default: jg
MinLength: '1'
AllowedPattern: "[a-zA-Z0-9]+"
EdgestoreReadCapacity:
Description: Read capacity of the edgestore table
Type: Number
Default: 12
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
EdgestoreWriteCapacity:
Description: Write capacity of the edgestore table
Type: Number
Default: 12
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
GraphindexReadCapacity:
Description: Read capacity of the graphindex table
Type: Number
Default: 9
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
GraphindexWriteCapacity:
Description: Write capacity of the graphindex table
Type: Number
Default: 9
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
JanusGraphIdsReadCapacity:
Description: Read capacity of the janusgraph_ids table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
JanusGraphIdsWriteCapacity:
Description: Write capacity of the janusgraph_ids table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemPropertiesReadCapacity:
Description: Read capacity of the system_properties table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemPropertiesWriteCapacity:
Description: Write capacity of the system_properties table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
TxlogReadCapacity:
Description: Read capacity of the txlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
TxlogWriteCapacity:
Description: Write capacity of the txlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemlogReadCapacity:
Description: Read capacity of the systemlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
SystemlogWriteCapacity:
Description: Write capacity of the systemlog table
Type: Number
Default: 1
MinValue: 1
MaxValue: 10000
ConstraintDescription: Must be between 1 and 10000
Resources:
Edgestore:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: EdgestoreReadCapacity
WriteCapacityUnits:
Ref: EdgestoreWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _edgestore
Graphindex:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: GraphindexReadCapacity
WriteCapacityUnits:
Ref: GraphindexWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _graphindex
JanusGraphIds:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: JanusGraphIdsReadCapacity
WriteCapacityUnits:
Ref: JanusGraphIdsWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _janusgraph_ids
SystemProperties:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: SystemPropertiesReadCapacity
WriteCapacityUnits:
Ref: SystemPropertiesWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _system_properties
Txlog:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: TxlogReadCapacity
WriteCapacityUnits:
Ref: TxlogWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _txlog
Systemlog:
Type: "AWS::DynamoDB::Table"
Properties:
AttributeDefinitions:
-
AttributeName: "hk"
AttributeType: "S"
KeySchema:
-
AttributeName: "hk"
KeyType: "HASH"
ProvisionedThroughput:
ReadCapacityUnits:
Ref: SystemlogReadCapacity
WriteCapacityUnits:
Ref: SystemlogWriteCapacity
TableName:
Fn::Join:
- ''
- - Ref: TablePrefix
- _systemlog
================================================
FILE: pom.xml
================================================
4.0.0com.amazonawsdynamodb-janusgraph-storage-backend1.2.0jarAmazon DynamoDB Storage Backend for JanusGraphhttps://github.com/awslabs/dynamodb-janusgraph-storage-backendThe Amazon DynamoDB Storage Backend for JanusGraph: Distributed Graph Database allows JanusGraph graphs to use DynamoDB as a storage backend.git@github.com:awslabs/dynamodb-janusgraph-storage-backend.gitjg0.2.0-1.2.0-Xms256m -Xmx1280m -XX:+HeapDumpOnOutOfMemoryError-Xms256m -Xmx768m -ea -XX:+HeapDumpOnOutOfMemoryErrorUTF-84567http://localhost:${dynamodb-local.port}1.81.11.3362.10.20.2.03.2.63.0.23.1.03.6.22.202.203.0.21.6.01.2.11.7.123.83.0.11.1.12.2.01.8.51.18.20.4.132.329.0-jreUTF-83.0.0-M1org.janusgraph.testcategory.MemoryTests,org.janusgraph.testcategory.PerformanceTests,org.janusgraph.testcategory.BrittleTests,org.janusgraph.testcategory.OrderedKeyStoreTests,org.janusgraph.testcategory.SerialTestsfalsefalse4.12Alexander Patrikalakisamcp@mit.eduhttps://www.linkedin.com/in/amcpatrix/enMatthew Sowdersmatthewsowders@gmail.comhttps://www.linkedin.com/in/matthewsowders/enMichael Rodaitismrodaitis@gmail.comZameer Meralizmerali@amazon.comJustin Panianpanianj@amazon.comAddison Slabaughacs254@cornell.eduJohn StephensonJohan Jacobsjohanjcbs@gmail.comhttps://www.linkedin.com/in/johanjcbs/Daniel Juedjue@phy6.net2014The Apache Software License, Version 2.0http://www.apache.org/licenses/LICENSE-2.0.txtcom.amazonawsaws-java-sdk-dynamodb${aws.java.sdk.version}org.janusgraphjanusgraph-core${janusgraph.version}com.google.code.findbugsjsr305org.janusgraphjanusgraph-test${janusgraph.version}testcom.codahale.metricsmetrics-core${metrics3.version}com.opencsvopencsv${opencsv.version}org.apache.commonscommons-lang3org.slf4jslf4j-log4j12${slf4j.version}testorg.apache.tinkerpopgremlin-core${tinkerpop.version}junitjunit${junit.version}testcommons-codeccommons-codec1.7commons-langcommons-lang2.6com.amazonawsaws-java-sdk-core${aws.java.sdk.version}log4jlog4j1.2.17testcommons-configurationcommons-configuration1.10runtimeorg.apache.commonscommons-lang33.3.1org.slf4jslf4j-api${slf4j.version}com.google.guavaguava${guava.version}org.mockitomockito-all${mockito.version}testorg.mockitomockito-core${mockito.version}testorg.hamcresthamcrest-coreorg.objenesisobjenesisorg.projectlomboklombok${lombok.version}com.google.code.findbugsfindbugs3.0.1com.google.code.findbugsjsr305com.fasterxml.jackson.corejackson-core${jackson.version}com.fasterxml.jackson.corejackson-databind${jackson.version}com.fasterxml.jackson.dataformatjackson-dataformat-cbor${jackson.version}junitjunit${junit.version}commons-iocommons-io${commonsio.version}commons-collectionscommons-collections3.2.2commons-codeccommons-codec1.7commons-loggingcommons-logging1.1.3com.codahale.metricsmetrics-core${metrics3.version}com.codahale.metricsmetrics-graphite${metrics3.version}com.codahale.metricsmetrics-ganglia${metrics3.version}org.slf4jslf4j-api${slf4j.version}com.google.guavaguava${guava.version}com.carrotsearchjunit-benchmarks0.7.0org.mockitomockito-core${mockito.version}org.codehaus.mojofindbugs-maven-plugin3.0.4MaxLowtruecheckorg.apache.maven.pluginsmaven-checkstyle-plugin2.17validatevalidatecheckstyle.xmlUTF-8truetruecheckorg.apache.maven.pluginsmaven-pmd-plugin3.8checkcpd-checkorg.apache.maven.pluginsmaven-dependency-plugin${dependency.plugin.version}analyzeanalyze-onlytruetrueorg.projectlombok:lombok:jar:${lombok.version}copy-dependenciesprocess-test-resourcescopy-dependencies${project.build.directory}/dependenciesfalsefalsetruemaven-compiler-plugin${maven.compiler.plugin.version}${jdk.version}${jdk.version}maven-surefire-plugin${maven.surefire.version}trueorg.apache.maven.pluginsmaven-resources-plugin${maven.resources.plugin.version}org.apache.maven.pluginsmaven-enforcer-plugin${maven-enforcer-plugin.version}enforceenforcedynamodb-janusgraph-dockercom.spotifydocker-maven-plugin${docker.maven.version}${project.basedir}/src/test/resources/dynamodb-janusgraph-dockerdynamodb-janusgraph-storage-backend-${project.version}.ziptruedynamodb-janusgraph/server${project.version}integration-testscom.googlecode.maven-download-plugindownload-maven-plugin${download.maven.plugin.version}install-dynamodb-localpre-integration-testwgethttps://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.ziptrue${project.build.directory}/dynamodb${download.skip.cache}${download.force.overwrite}com.bazaarvoice.maven.pluginsprocess-exec-maven-plugin0.4dynamodb-localpre-integration-teststartdynamodb-local1java-Djava.library.path=dynamodb/DynamoDBLocal_lib-jardynamodb/DynamoDBLocal.jar-inMemory-port${dynamodb-local.port}-sharedDbstop-jar-processpost-integration-teststop-allorg.apache.maven.pluginsmaven-failsafe-plugin${maven.failsafe.version}default-integration-testsintegration-testverify${test.excluded.groups},${exclude.category}${include.category}false110000truesrc/test/resources/dynamodb-local.properties${dynamodb-local.endpoint}download-janusgraph-server-zipcom.googlecode.maven-download-plugindownload-maven-plugin${download.maven.plugin.version}download-janusgraph-server-zipwgethttps://github.com/JanusGraph/janusgraph/releases/download/v${janusgraph.version}/janusgraph-${janusgraph.version}-hadoop2.zipfalse${project.build.directory}/../server${download.skip.cache}${download.force.overwrite}start-dynamodb-localcom.googlecode.maven-download-plugindownload-maven-plugin${download.maven.plugin.version}install-dynamodb_localwgethttps://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.ziptrue${project.build.directory}/dynamodb${download.skip.cache}${download.force.overwrite}org.codehaus.mojoexec-maven-plugin${exec.maven.plugin.version}test${project.build.directory}/dynamodbjava-Djava.library.path=${project.build.directory}/dynamodb/DynamoDBLocal_lib-jar${project.build.directory}/dynamodb/DynamoDBLocal.jar-inMemory-port${dynamodb-local.port}-sharedDbexec
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/AbstractDynamoDbStore.java
================================================
/*
* Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.DeleteItemRequest;
import com.amazonaws.services.dynamodbv2.model.GetItemRequest;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import com.amazonaws.services.dynamodbv2.model.QueryRequest;
import com.amazonaws.services.dynamodbv2.model.ReturnConsumedCapacity;
import com.amazonaws.services.dynamodbv2.model.ScanRequest;
import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest;
import lombok.Getter;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.diskstorage.Entry;
import org.janusgraph.diskstorage.StaticBuffer;
import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation;
import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
import org.janusgraph.diskstorage.locking.TemporaryLockingException;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.common.util.concurrent.ExecutionError;
import com.google.common.util.concurrent.UncheckedExecutionException;
import lombok.extern.slf4j.Slf4j;
/**
* The base class for the SINGLE and MULTI implementations of the Amazon DynamoDB Storage Backend
* for JanusGraph distributed store type.
* @author Matthew Sowders
* @author Alexander Patrikalakis
*
*/
@Slf4j
public abstract class AbstractDynamoDbStore implements AwsStore {
protected final Client client;
@Getter
private final String tableName;
private final DynamoDBStoreManager manager;
@Getter
private final String name;
private final boolean forceConsistentRead;
/**
* The key column local lock cache maps key-column pairs to the DynamoDbStoreTransaction that first
* acquired a lock on those key-column pairs.
*/
private final Cache, DynamoDbStoreTransaction> keyColumnLocalLocks;
private enum ReportingRemovalListener implements RemovalListener, DynamoDbStoreTransaction> {
INSTANCE;
@Override
public void onRemoval(final RemovalNotification, DynamoDbStoreTransaction> notice) {
log.trace("Expiring {} in tx {} because of {}", notice.getKey().toString(), notice.getValue().toString(), notice.getCause());
}
}
protected void mutateOneKey(final StaticBuffer key, final KCVMutation mutation, final StoreTransaction txh) throws BackendException {
manager.mutateMany(Collections.singletonMap(name, Collections.singletonMap(key, mutation)), txh);
}
protected UpdateItemRequest createUpdateItemRequest() {
return new UpdateItemRequest()
.withTableName(tableName)
.withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
}
protected GetItemRequest createGetItemRequest() {
return new GetItemRequest()
.withTableName(tableName)
.withConsistentRead(forceConsistentRead)
.withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
}
protected DeleteItemRequest createDeleteItemRequest() {
return new DeleteItemRequest()
.withTableName(tableName)
.withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
}
protected QueryRequest createQueryRequest() {
return new QueryRequest()
.withTableName(tableName)
.withConsistentRead(forceConsistentRead)
.withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
}
protected ScanRequest createScanRequest() {
return new ScanRequest().withTableName(tableName)
.withConsistentRead(forceConsistentRead)
.withLimit(client.scanLimit(tableName))
.withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
}
AbstractDynamoDbStore(final DynamoDBStoreManager manager, final String prefix, final String storeName) {
this.manager = manager;
this.client = this.manager.getClient();
this.name = storeName;
this.tableName = prefix + "_" + storeName;
this.forceConsistentRead = client.isForceConsistentRead();
final CacheBuilder, DynamoDbStoreTransaction> builder = CacheBuilder.newBuilder().concurrencyLevel(client.getDelegate().getMaxConcurrentUsers())
.expireAfterWrite(manager.getLockExpiresDuration().toMillis(), TimeUnit.MILLISECONDS)
.removalListener(ReportingRemovalListener.INSTANCE);
this.keyColumnLocalLocks = builder.build();
}
/**
* Creates the schemata for the DynamoDB table or tables each store requires.
* Implementations should override and reuse this logic
* @return a create table request appropriate for the schema of the selected implementation.
*/
public CreateTableRequest getTableSchema() {
return new CreateTableRequest()
.withTableName(tableName)
.withProvisionedThroughput(new ProvisionedThroughput(client.readCapacity(tableName),
client.writeCapacity(tableName)));
}
@Override
public final void ensureStore() throws BackendException {
log.debug("Entering ensureStore table:{}", tableName);
client.getDelegate().createTableAndWaitForActive(getTableSchema());
}
@Override
public final void deleteStore() throws BackendException {
log.debug("Entering deleteStore name:{}", name);
client.getDelegate().deleteTable(getTableSchema().getTableName());
//block until the tables are actually deleted
client.getDelegate().ensureTableDeleted(getTableSchema().getTableName());
}
@Override
public void acquireLock(final StaticBuffer key, final StaticBuffer column, final StaticBuffer expectedValue, final StoreTransaction txh) throws BackendException {
final DynamoDbStoreTransaction tx = DynamoDbStoreTransaction.getTx(txh);
final Pair keyColumn = Pair.of(key, column);
final DynamoDbStoreTransaction existing;
try {
existing = keyColumnLocalLocks.get(keyColumn, () -> tx);
} catch (ExecutionException | UncheckedExecutionException | ExecutionError e) {
throw new TemporaryLockingException("Unable to acquire lock", e);
}
if (null != existing && tx != existing) {
throw new TemporaryLockingException(String.format("tx %s already locked key-column %s when tx %s tried to lock", existing.toString(), keyColumn.toString(), tx.toString()));
}
// Titan's locking expects that only the first expectedValue for a given key/column should be used
tx.putKeyColumnOnlyIfItIsNotYetChangedInTx(this, key, column, expectedValue);
}
@Override
public void close() throws BackendException {
log.debug("Closing table:{}", tableName);
}
String encodeKeyForLog(final StaticBuffer key) {
if (null == key) {
return "";
}
return Constants.HEX_PREFIX + Hex.encodeHexString(key.asByteBuffer().array());
}
String encodeForLog(final List> columns) {
return columns.stream()
.map(obj -> {
if (obj instanceof StaticBuffer) {
return (StaticBuffer) obj;
} else if (obj instanceof Entry) {
return ((Entry) obj).getColumn();
} else {
return null;
}
})
.map(this::encodeKeyForLog)
.collect(Collectors.joining(",", "[", "]"));
}
@Override
public int hashCode() {
return tableName.hashCode();
}
@Override
public boolean equals(final Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (obj.getClass() != getClass()) {
return false;
}
final AbstractDynamoDbStore rhs = (AbstractDynamoDbStore) obj;
return new EqualsBuilder().append(tableName, rhs.tableName).isEquals();
}
@Override
public String toString() {
return this.getClass().getName() + ":" + getTableName();
}
protected String encodeForLog(final SliceQuery query) {
return "slice[rk:" + encodeKeyForLog(query.getSliceStart()) + " -> " + encodeKeyForLog(query.getSliceEnd()) + " limit:" + query.getLimit() + "]";
}
protected String encodeForLog(final KeySliceQuery query) {
return "keyslice[hk:" + encodeKeyForLog(query.getKey()) + " " + "rk:" + encodeKeyForLog(query.getSliceStart()) + " -> " + encodeKeyForLog(query.getSliceEnd()) + " limit:"
+ query.getLimit() + "]";
}
void releaseLock(final StaticBuffer key, final StaticBuffer column) {
keyColumnLocalLocks.invalidate(Pair.of(key, column));
}
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/AwsStore.java
================================================
/*
* Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import java.util.Collection;
import java.util.Map;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.diskstorage.StaticBuffer;
import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation;
import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.MutateWorker;
/**
* Responsible for communicating with a single AWS backing store table.
*
* @author Matthew Sowders
* @author Alexander Patrikalakis
*
*/
public interface AwsStore extends KeyColumnValueStore {
/**
* Creates the KCV store and underlying DynamoDB tables.
* @throws BackendException if unable to ensure the underlying store
*/
void ensureStore() throws BackendException;
/**
* Deletes the KCV store and underlying DynamoDB tables.
* @throws BackendException
*/
void deleteStore() throws BackendException;
/**
* Titan relies on static store names to be used, but we want the ability to
* have multiple graphs in a single region, so prepend a configurable prefix to the
* underlying table names of each graph and get the DynamoDB table name with this method
*
* @return the table name corresponding to the KCVStore
*/
String getTableName();
/**
* Creates workers whose job it is to commit the actual mutations in the given mutation map.
* @param mutationMap
* @param txh
* @return a collection of MutateWorker objects that when executed will commit all changes specified by mutationMap
*/
Collection createMutationWorkers(Map mutationMap,
DynamoDbStoreTransaction txh);
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/BackendDataModel.java
================================================
/*
* Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* Creates a store backend based on configuration.
*
* @author Matthew Sowders
*
*/
@RequiredArgsConstructor
public enum BackendDataModel {
SINGLE("Single") {
@Override
public AwsStore createStoreBackend(final DynamoDBStoreManager manager, final String prefix, final String name) {
return new DynamoDbSingleRowStore(manager, prefix, name);
}
},
MULTI("Multiple") {
@Override
public AwsStore createStoreBackend(final DynamoDBStoreManager manager, final String prefix, final String name) {
return new DynamoDbStore(manager, prefix, name);
}
};
@Getter
private final String camelCaseName;
public abstract AwsStore createStoreBackend(DynamoDBStoreManager manager, String prefix, String name);
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/BackendNotFoundException.java
================================================
/*
* Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import org.janusgraph.diskstorage.PermanentBackendException;
/**
* Interpretation of DynamoDB ResourceNotFoundException
* @author Alexander Patrikalakis
*
*/
public class BackendNotFoundException extends PermanentBackendException {
private static final long serialVersionUID = 5595152932709453493L;
public BackendNotFoundException(final String msg, final Throwable cause) {
super(msg, cause);
}
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/BackendRuntimeException.java
================================================
/*
* Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import org.janusgraph.diskstorage.BackendException;
/**
* This class wraps around StorageExceptions in the Amazon DynamoDB Storage Backend for Titan,
* a checked exception
* @author Alexander Patrikalakis
*
*/
public class BackendRuntimeException extends RuntimeException {
BackendRuntimeException(final String str) {
super(str);
}
public BackendRuntimeException(final BackendException e) {
super(e);
}
public BackendException getBackendException() {
final Throwable throwable = super.getCause();
if (throwable instanceof BackendException) {
return (BackendException) throwable;
}
return null;
}
private static final long serialVersionUID = 6184087040805925812L;
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/Client.java
================================================
/*
* Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
* Portions copyright Titan: Distributed Graph Database - Copyright 2012 and onwards Aurelius.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.NonNull;
import org.janusgraph.diskstorage.configuration.Configuration;
import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.util.concurrent.RateLimiter;
import com.google.common.util.concurrent.RateLimiterCreator;
/**
* Operations setting up the DynamoDB client.
*
* @author Matthew Sowders
* @author Alexander Patrikalakis
*
*/
public class Client {
private static final String VALIDATE_CREDENTIALS_CLASS_NAME = "Must provide either an AWSCredentials or AWSCredentialsProvider fully qualified class name";
private static final double DEFAULT_BURST_BUCKET_SIZE_IN_SECONDS = 300.0;
private final Map capacityRead = new HashMap<>();
private final Map capacityWrite = new HashMap<>();
private final Map dataModelMap = new HashMap<>();
@Getter(AccessLevel.PACKAGE)
private final boolean forceConsistentRead;
@Getter(AccessLevel.PACKAGE)
private final boolean enableParallelScan;
private final Map scanLimitMap = new HashMap<>();
@Getter
private final DynamoDbDelegate delegate;
@Getter
private final String prefix;
public Client(final Configuration config) {
final String credentialsClassName = config.get(Constants.DYNAMODB_CREDENTIALS_CLASS_NAME);
final Class> clazz;
try {
clazz = Class.forName(credentialsClassName);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException(VALIDATE_CREDENTIALS_CLASS_NAME, e);
}
final String[] credentialsConstructorArgsValues = config.get(Constants.DYNAMODB_CREDENTIALS_CONSTRUCTOR_ARGS);
final List filteredArgList = new ArrayList<>();
for (Object obj : credentialsConstructorArgsValues) {
final String str = obj.toString();
if (!str.isEmpty()) {
filteredArgList.add(str);
}
}
final AWSCredentialsProvider credentialsProvider;
if (AWSCredentials.class.isAssignableFrom(clazz)) {
final AWSCredentials credentials = createCredentials(clazz, filteredArgList.toArray(new String[filteredArgList.size()]));
credentialsProvider = new AWSStaticCredentialsProvider(credentials);
} else if (AWSCredentialsProvider.class.isAssignableFrom(clazz)) {
credentialsProvider = createCredentialsProvider(clazz, credentialsConstructorArgsValues);
} else {
throw new IllegalArgumentException(VALIDATE_CREDENTIALS_CLASS_NAME);
}
//begin adaptation of constructor at
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L77
final ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.withConnectionTimeout(config.get(Constants.DYNAMODB_CLIENT_CONN_TIMEOUT))
.withConnectionTTL(config.get(Constants.DYNAMODB_CLIENT_CONN_TTL))
.withMaxConnections(config.get(Constants.DYNAMODB_CLIENT_MAX_CONN))
.withMaxErrorRetry(config.get(Constants.DYNAMODB_CLIENT_MAX_ERROR_RETRY))
.withGzip(config.get(Constants.DYNAMODB_CLIENT_USE_GZIP))
.withReaper(config.get(Constants.DYNAMODB_CLIENT_USE_REAPER))
.withUserAgentSuffix(config.get(Constants.DYNAMODB_CLIENT_USER_AGENT))
.withSocketTimeout(config.get(Constants.DYNAMODB_CLIENT_SOCKET_TIMEOUT))
.withSocketBufferSizeHints(
config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_SEND_HINT),
config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_RECV_HINT))
.withProxyDomain(config.get(Constants.DYNAMODB_CLIENT_PROXY_DOMAIN))
.withProxyWorkstation(config.get(Constants.DYNAMODB_CLIENT_PROXY_WORKSTATION))
.withProxyHost(config.get(Constants.DYNAMODB_CLIENT_PROXY_HOST))
.withProxyPort(config.get(Constants.DYNAMODB_CLIENT_PROXY_PORT))
.withProxyUsername(config.get(Constants.DYNAMODB_CLIENT_PROXY_USERNAME))
.withProxyPassword(config.get(Constants.DYNAMODB_CLIENT_PROXY_PASSWORD));
forceConsistentRead = config.get(Constants.DYNAMODB_FORCE_CONSISTENT_READ);
//end adaptation of constructor at
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L77
enableParallelScan = config.get(Constants.DYNAMODB_ENABLE_PARALLEL_SCAN);
prefix = config.get(Constants.DYNAMODB_TABLE_PREFIX);
final String metricsPrefix = config.get(Constants.DYNAMODB_METRICS_PREFIX);
final long maxRetries = config.get(Constants.DYNAMODB_MAX_SELF_THROTTLED_RETRIES);
Preconditions.checkArgument(maxRetries >= 0,
Constants.DYNAMODB_MAX_SELF_THROTTLED_RETRIES.getName() + " must be at least 0");
final long retryMillis = config.get(Constants.DYNAMODB_INITIAL_RETRY_MILLIS);
Preconditions.checkArgument(retryMillis > 0,
Constants.DYNAMODB_INITIAL_RETRY_MILLIS.getName() + " must be at least 1");
final double controlPlaneRate = config.get(Constants.DYNAMODB_CONTROL_PLANE_RATE);
Preconditions.checkArgument(controlPlaneRate > 0,
"must have a positive control plane rate");
final RateLimiter controlPlaneRateLimiter = RateLimiterCreator.createBurstingLimiter(controlPlaneRate,
DEFAULT_BURST_BUCKET_SIZE_IN_SECONDS);
final Map readRateLimit = new HashMap<>();
final Map writeRateLimit = new HashMap<>();
final Set storeNames = new HashSet<>(Constants.REQUIRED_BACKEND_STORES);
storeNames.add(config.get(GraphDatabaseConfiguration.IDS_STORE_NAME));
storeNames.addAll(config.getContainedNamespaces(Constants.DYNAMODB_STORES_NAMESPACE));
storeNames.forEach(storeName -> setupStore(config, readRateLimit, writeRateLimit, storeName));
delegate = new DynamoDbDelegate(JanusGraphConfigUtil.getNullableConfigValue(config, Constants.DYNAMODB_CLIENT_ENDPOINT),
JanusGraphConfigUtil.getNullableConfigValue(config, Constants.DYNAMODB_CLIENT_SIGNING_REGION),
credentialsProvider,
clientConfig, config, readRateLimit, writeRateLimit, maxRetries, retryMillis, prefix, metricsPrefix, controlPlaneRateLimiter);
}
private void setupStore(final Configuration config,
final Map readRateLimit, final Map writeRateLimit, final String store) {
final String dataModel = config.get(Constants.STORES_DATA_MODEL, store);
final int scanLimit = config.get(Constants.STORES_SCAN_LIMIT, store);
final long readCapacity = config.get(Constants.STORES_INITIAL_CAPACITY_READ, store);
final long writeCapacity = config.get(Constants.STORES_INITIAL_CAPACITY_WRITE, store);
final double readRate = config.get(Constants.STORES_READ_RATE_LIMIT, store);
final double writeRate = config.get(Constants.STORES_WRITE_RATE_LIMIT, store);
final String actualTableName = prefix + "_" + store;
this.dataModelMap.put(store, BackendDataModel.valueOf(dataModel));
this.capacityRead.put(actualTableName, readCapacity);
this.capacityWrite.put(actualTableName, writeCapacity);
readRateLimit.put(actualTableName, RateLimiterCreator.createBurstingLimiter(readRate, DEFAULT_BURST_BUCKET_SIZE_IN_SECONDS));
writeRateLimit.put(actualTableName, RateLimiterCreator.createBurstingLimiter(writeRate, DEFAULT_BURST_BUCKET_SIZE_IN_SECONDS));
this.scanLimitMap.put(actualTableName, scanLimit);
}
long readCapacity(@NonNull final String tableName) {
return capacityRead.get(tableName);
}
long writeCapacity(@NonNull final String tableName) {
return capacityWrite.get(tableName);
}
BackendDataModel dataModel(final String storeName) {
return dataModelMap.get(storeName);
}
int scanLimit(final String tableName) {
return scanLimitMap.get(tableName);
}
private static AWSCredentialsProvider createCredentialsProvider(final Class> clazz, final String[] credentialsProviderConstructorArgs) {
return (AWSCredentialsProvider) createInstance(clazz, credentialsProviderConstructorArgs);
}
private static AWSCredentials createCredentials(final Class> clazz, final String[] credentialsConstructorArgs) {
return (AWSCredentials) createInstance(clazz, credentialsConstructorArgs);
}
private static Object createInstance(final Class> clazz, final String[] constructorArgs) {
final Class>[] constructorTypes;
String[] actualArgs = constructorArgs;
if (null == constructorArgs) {
constructorTypes = new Class>[0];
} else if (constructorArgs.length == 1 && Strings.isNullOrEmpty(constructorArgs[0])) {
// Special case for empty constructors
actualArgs = new String[0];
constructorTypes = new Class>[0];
} else {
constructorTypes = new Class>[constructorArgs.length];
for (int i = 0; i < constructorArgs.length; i++) {
constructorTypes[i] = String.class;
}
}
final Constructor> constructor;
try {
constructor = clazz.getConstructor(constructorTypes);
} catch (NoSuchMethodException | SecurityException e) {
throw new IllegalArgumentException("Cannot access constructor:" + clazz.getCanonicalName() + "(" + constructorTypes.length + ")", e);
}
final Object instance;
try {
instance = constructor.newInstance((Object[]) actualArgs);
} catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
throw new IllegalArgumentException("Cannot create new instance:" + clazz.getCanonicalName(), e);
}
return instance;
}
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/Constants.java
================================================
/*
* Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
* Portions copyright Titan: Distributed Graph Database - Copyright 2012 and onwards Aurelius.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import static org.janusgraph.diskstorage.configuration.ConfigOption.Type.FIXED;
import static org.janusgraph.diskstorage.configuration.ConfigOption.Type.LOCAL;
import java.util.List;
import org.janusgraph.diskstorage.Backend;
import org.janusgraph.diskstorage.configuration.ConfigNamespace;
import org.janusgraph.diskstorage.configuration.ConfigOption;
import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration;
import com.amazonaws.ClientConfiguration;
import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableList;
/**
* Constants for the DynamoDB backend.
*
* @author Matthew Sowders
* @author Alexander Patrikalakis
*
*/
public final class Constants {
private Constants() { }
//begin adaptation of
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L26
public static final String JANUSGRAPH_VALUE = "v";
public static final String JANUSGRAPH_HASH_KEY = "hk";
public static final String JANUSGRAPH_RANGE_KEY = "rk";
//end adaptation of
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L26
public static final String HEX_PREFIX = "0x";
public static final String JANUSGRAPH_USER_AGENT = "dynamodb-janusgraph010-storage-backend_1.0.0";
public static final List REQUIRED_BACKEND_STORES = ImmutableList.of(Backend.EDGESTORE_NAME,
Backend.INDEXSTORE_NAME,
Backend.SYSTEM_TX_LOG_NAME,
Backend.SYSTEM_MGMT_LOG_NAME,
GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME);
public static final ConfigNamespace DYNAMODB_CONFIGURATION_NAMESPACE =
new ConfigNamespace(GraphDatabaseConfiguration.STORAGE_NS, "dynamodb",
"DynamoDB storage options", false /*isUmbrella*/);
public static final ConfigNamespace DYNAMODB_STORES_NAMESPACE =
new ConfigNamespace(DYNAMODB_CONFIGURATION_NAMESPACE, "stores",
"DynamoDB KCV store options", true /*isUmbrella*/);
public static final ConfigNamespace DYNAMODB_CLIENT_NAMESPACE =
new ConfigNamespace(DYNAMODB_CONFIGURATION_NAMESPACE,
"client", "DynamoDB client options", false /*isUmbrella*/);
public static final ConfigNamespace DYNAMODB_CLIENT_PROXY_NAMESPACE =
new ConfigNamespace(DYNAMODB_CLIENT_NAMESPACE,
"proxy", "DynamoDB client proxy options", false /*isUmbrella*/);
public static final ConfigNamespace DYNAMODB_CLIENT_SOCKET_NAMESPACE =
new ConfigNamespace(DYNAMODB_CLIENT_NAMESPACE, "socket", "DynamoDB client socket options", false /*isUmbrella*/);
public static final ConfigNamespace DYNAMODB_CLIENT_EXECUTOR_NAMESPACE =
new ConfigNamespace(DYNAMODB_CLIENT_NAMESPACE,
"executor", "DynamoDB client executor options", false /*isUmbrella*/);
public static final ConfigNamespace DYNAMODB_CLIENT_CREDENTIALS_NAMESPACE =
new ConfigNamespace(DYNAMODB_CLIENT_NAMESPACE, "credentials", "DynamoDB client credentials options",
false /*isUmbrella*/);
public static final ConfigOption DYNAMODB_TABLE_PREFIX = new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE,
"prefix", "A prefix to put before the JanusGraph table name. "
+ "This allows clients to have multiple graphs on the same AWS DynamoDB account.",
LOCAL, "jg");
public static final ConfigOption DYNAMODB_METRICS_PREFIX = new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE, "metrics-prefix",
"Prefix on the codahale metric names emitted by DynamoDbDelegate.",
LOCAL, "d");
public static final ConfigOption DYNAMODB_ENABLE_PARALLEL_SCAN =
new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE, "enable-parallel-scans",
"This feature enables scans to run in parallel, which should decrease the total blocking time "
+ "spent when iterating over large sets of vertices. "
+ "WARNING: while this feature is enabled JanusGraph's OLAP libraries are NOT supported."
+ "The JanusGraph-Hadoop implementations of OLAP rely on consistent scan orders across multiple scans, "
+ "which cannot be guaranteed when scans are run in parallel",
LOCAL, false);
public static final ConfigOption STORES_DATA_MODEL =
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "data-model",
"SINGLE Means that all the values for a given key are put into a single DynamoDB item. "
+ "A SINGLE is efficient because all the updates for a single key can be done atomically. "
+ "However, the trade-off is that DynamoDB has a 400k limit per item so it cannot hold much data. "
+ "MULTI Means that each 'column' is used as a range key in DynamoDB so a key can span multiple items. "
+ "A MULTI implementation is slightly less efficient than SINGLE because it must use DynamoDB Query "
+ "rather than a direct lookup. It is HIGHLY recommended to use MULTI for edgestore unless your graph has "
+ "very low max degree.",
FIXED, BackendDataModel.MULTI.name());
public static final ConfigOption STORES_SCAN_LIMIT =
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "scan-limit",
"The maximum number of items to evaluate (not necessarily the number of matching items). "
+ "If DynamoDB processes the number of items up to the limit while processing the results, it stops "
+ "the operation and returns the matching values up to that point, and a key in LastEvaluatedKey to apply "
+ "in a subsequent operation, so that you can pick up where you left off. Also, if the processed data set "
+ "size exceeds 1 MB before DynamoDB reaches this limit, it stops the operation and returns the matching "
+ "values up to the limit, and a key in LastEvaluatedKey to apply in a subsequent operation to continue "
+ "the operation.",
LOCAL, 10000);
public static final ConfigOption DYNAMODB_USE_NATIVE_LOCKING = new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE,
"native-locking", "Set this to false if you need to use JanusGraph's locking mechanism for remote lock expiry.",
FIXED, true);
//begin adaptation of the following block up until line 63
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L29
public static final ConfigOption DYNAMODB_FORCE_CONSISTENT_READ =
new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE, "force-consistent-read",
"This feature sets the force consistent read property on dynamodb calls.",
LOCAL, true);
public static final ConfigOption STORES_INITIAL_CAPACITY_READ =
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "initial-capacity-read",
"Define the initial read capacity for a given dynamodb table.",
LOCAL, 4L);
public static final ConfigOption STORES_INITIAL_CAPACITY_WRITE =
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "initial-capacity-write",
"Define the initial write capacity for a given dynamodb table.",
LOCAL, 4L);
public static final ConfigOption STORES_READ_RATE_LIMIT =
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "read-rate",
"The max number of reads per second.",
LOCAL, 4.0);
public static final ConfigOption STORES_WRITE_RATE_LIMIT =
new ConfigOption<>(Constants.DYNAMODB_STORES_NAMESPACE, "write-rate",
"Used to throttle write rate of given table. The max number of writes per second.",
LOCAL, 4.0);
public static final ConfigOption DYNAMODB_CLIENT_CONN_TIMEOUT =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "connection-timeout",
"The amount of time to wait (in milliseconds) when initially establishing a connection before giving up and timing out.", //
LOCAL, ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT);
public static final ConfigOption DYNAMODB_CLIENT_CONN_TTL =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "connection-ttl",
"The expiration time (in milliseconds) for a connection in the connection pool.",
LOCAL, ClientConfiguration.DEFAULT_CONNECTION_TTL);
public static final ConfigOption DYNAMODB_CLIENT_MAX_CONN =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "connection-max",
"The maximum number of allowed open HTTP connections.",
LOCAL, ClientConfiguration.DEFAULT_MAX_CONNECTIONS);
public static final ConfigOption DYNAMODB_CLIENT_MAX_ERROR_RETRY =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "retry-error-max",
"The maximum number of retry attempts for failed retryable "
+ "requests (ex: 5xx error responses from services).",
LOCAL, 0);
public static final ConfigOption DYNAMODB_CLIENT_USE_GZIP =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "use-gzip",
"Sets whether gzip compression should be used.",
LOCAL, ClientConfiguration.DEFAULT_USE_GZIP);
public static final ConfigOption DYNAMODB_CLIENT_USE_REAPER =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "use-reaper",
"Sets whether the IdleConnectionReaper is to be started as a daemon thread.",
LOCAL, ClientConfiguration.DEFAULT_USE_REAPER);
public static final ConfigOption DYNAMODB_CLIENT_USER_AGENT =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "user-agent",
"The HTTP user agent header to send with all requests.",
LOCAL, JANUSGRAPH_USER_AGENT);
public static final ConfigOption DYNAMODB_CLIENT_ENDPOINT =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "endpoint",
"Sets the service endpoint to use for connecting to DynamoDB.",
LOCAL, String.class);
public static final ConfigOption DYNAMODB_CLIENT_SIGNING_REGION =
new ConfigOption<>(DYNAMODB_CLIENT_NAMESPACE, "signing-region",
"Sets the signing region to use for signing requests to DynamoDB. Required.",
LOCAL, String.class);
public static final ConfigOption DYNAMODB_CLIENT_PROXY_DOMAIN =
new ConfigOption<>(DYNAMODB_CLIENT_PROXY_NAMESPACE, "domain",
"The optional Windows domain name for configuration an NTLM proxy.",
LOCAL, "", Predicates.alwaysTrue());
public static final ConfigOption DYNAMODB_CLIENT_PROXY_WORKSTATION =
new ConfigOption<>(DYNAMODB_CLIENT_PROXY_NAMESPACE, "workstation",
"The optional Windows workstation name for configuring NTLM proxy support.",
LOCAL, "", Predicates.alwaysTrue());
public static final ConfigOption DYNAMODB_CLIENT_PROXY_HOST =
new ConfigOption<>(DYNAMODB_CLIENT_PROXY_NAMESPACE, "host",
"The optional proxy host the client will connect through.",
LOCAL, "", Predicates.alwaysTrue());
public static final ConfigOption DYNAMODB_CLIENT_PROXY_PORT =
new ConfigOption<>(DYNAMODB_CLIENT_PROXY_NAMESPACE, "port",
"The optional proxy port the client will connect through.",
LOCAL, 0, Predicates.alwaysTrue());
public static final ConfigOption DYNAMODB_CLIENT_PROXY_USERNAME =
new ConfigOption<>(DYNAMODB_CLIENT_PROXY_NAMESPACE, "username",
"The optional proxy user name to use if connecting through a proxy.",
LOCAL, "", Predicates.alwaysTrue());
public static final ConfigOption DYNAMODB_CLIENT_PROXY_PASSWORD =
new ConfigOption<>(DYNAMODB_CLIENT_PROXY_NAMESPACE, "password",
"The optional proxy password to use when connecting through a proxy.",
LOCAL, "", Predicates.alwaysTrue());
public static final ConfigOption DYNAMODB_CLIENT_SOCKET_BUFFER_SEND_HINT =
new ConfigOption<>(DYNAMODB_CLIENT_SOCKET_NAMESPACE, "buffer-send-hint",
"The optional size hint (in bytes) for the low level TCP send buffer.",
LOCAL, 1048576);
public static final ConfigOption DYNAMODB_CLIENT_SOCKET_BUFFER_RECV_HINT =
new ConfigOption<>(DYNAMODB_CLIENT_SOCKET_NAMESPACE, "buffer-recv-hint",
"The optional size hints (in bytes) for the low level TCP receive buffer.",
LOCAL, 1048576);
public static final ConfigOption DYNAMODB_CLIENT_SOCKET_TIMEOUT =
new ConfigOption<>(DYNAMODB_CLIENT_SOCKET_NAMESPACE, "timeout",
"The amount of time to wait (in milliseconds) for data to be transfered over an established, "
+ "open connection before the connection times out and is closed.",
LOCAL, ClientConfiguration.DEFAULT_SOCKET_TIMEOUT);
public static final ConfigOption DYNAMODB_CLIENT_SOCKET_TCP_KEEP_ALIVE =
new ConfigOption<>(DYNAMODB_CLIENT_SOCKET_NAMESPACE, "tcp-keep-alive",
"Sets whether or not to enable TCP KeepAlive support at the socket level. Not used at the moment.",
LOCAL, false);
public static final ConfigOption DYNAMODB_CLIENT_EXECUTOR_CORE_POOL_SIZE =
new ConfigOption<>(DYNAMODB_CLIENT_EXECUTOR_NAMESPACE, "core-pool-size",
"The core number of threads for the DynamoDB async client.",
LOCAL, 25);
public static final ConfigOption DYNAMODB_CLIENT_EXECUTOR_MAX_POOL_SIZE =
new ConfigOption<>(DYNAMODB_CLIENT_EXECUTOR_NAMESPACE, "max-pool-size",
"The maximum allowed number of threads for the DynamoDB async client.",
LOCAL, 50);
public static final ConfigOption DYNAMODB_CLIENT_EXECUTOR_KEEP_ALIVE =
new ConfigOption<>(DYNAMODB_CLIENT_EXECUTOR_NAMESPACE, "keep-alive",
"The time limit for which threads may remain idle before being terminated for the DynamoDB async client.",
LOCAL, 60000L);
//end adaptation of the following block up until line 63
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L29
public static final ConfigOption DYNAMODB_CLIENT_EXECUTOR_QUEUE_MAX_LENGTH =
new ConfigOption<>(DYNAMODB_CLIENT_EXECUTOR_NAMESPACE, "max-queue-length",
"The maximum size of the executor queue before requests start getting run in the caller.",
LOCAL, 1024);
public static final ConfigOption DYNAMODB_MAX_SELF_THROTTLED_RETRIES =
new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE, "max-self-throttled-retries",
"The max number of retries to use when DynamoDB throws temporary failure exceptions",
LOCAL, 60L);
public static final ConfigOption DYNAMODB_INITIAL_RETRY_MILLIS =
new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE, "initial-retry-millis",
"The initial retry time (in milliseconds) to use during exponential backoff between DynamoDB requests",
LOCAL, 25L);
public static final ConfigOption DYNAMODB_CONTROL_PLANE_RATE =
new ConfigOption<>(DYNAMODB_CONFIGURATION_NAMESPACE, "control-plane-rate",
"The maximum rate at which control plane requests (CreateTable, UpdateTable, DeleteTable, ListTables, "
+ "DescribeTable) are issued.",
LOCAL, 10.0);
public static final ConfigOption DYNAMODB_CLIENT_EXECUTOR_MAX_CONCURRENT_OPERATIONS =
new ConfigOption<>(DYNAMODB_CLIENT_EXECUTOR_NAMESPACE, "max-concurrent-operations",
"The expected number of threads expected to be using a single TitanGraph instance. "
+ "Used to allocate threads to batch operations", //
LOCAL, 1);
public static final ConfigOption DYNAMODB_CREDENTIALS_CLASS_NAME =
new ConfigOption<>(DYNAMODB_CLIENT_CREDENTIALS_NAMESPACE, "class-name",
"Specify the fully qualified class that implements AWSCredentialsProvider or AWSCredentials.",
LOCAL, "com.amazonaws.auth.BasicAWSCredentials");
public static final ConfigOption DYNAMODB_CREDENTIALS_CONSTRUCTOR_ARGS =
new ConfigOption<>(DYNAMODB_CLIENT_CREDENTIALS_NAMESPACE, "constructor-args",
"Comma separated list of strings to pass to the credentials constructor.",
LOCAL, new String[] {
"accessKey", "secretKey"
});
// DynamoDB doesn't allow empty binary values.
public static final String EMPTY_BUFFER_PLACEHOLDER = "EMPTY";
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/DynamoDBStoreManager.java
================================================
/*
* Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import java.io.IOException;
import java.net.URL;
import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import com.amazonaws.services.dynamodbv2.model.ListTablesRequest;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.diskstorage.BaseTransactionConfig;
import org.janusgraph.diskstorage.PermanentBackendException;
import org.janusgraph.diskstorage.StaticBuffer;
import org.janusgraph.diskstorage.StoreMetaData.Container;
import org.janusgraph.diskstorage.common.DistributedStoreManager;
import org.janusgraph.diskstorage.configuration.Configuration;
import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation;
import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore;
import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager;
import org.janusgraph.diskstorage.keycolumnvalue.KeyRange;
import org.janusgraph.diskstorage.keycolumnvalue.StandardStoreFeatures;
import org.janusgraph.diskstorage.keycolumnvalue.StandardStoreFeatures.Builder;
import org.janusgraph.diskstorage.keycolumnvalue.StoreFeatures;
import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
import org.janusgraph.diskstorage.util.time.TimestampProviders;
import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.MutateWorker;
import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
/**
* The JanusGraph manager for the Amazon DynamoDB Storage Backend for JanusGraph. Opens AwsStores. Tracks implemented
* features. Implements mutateMany used by the concrete implementations.
* @author Matthew Sowders
* @author Alexander Patrikalakis
*
*/
@Slf4j
public class DynamoDBStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager {
private static final int DEFAULT_PORT = 8080;
@VisibleForTesting
@Getter
Client client;
private final DynamoDbStoreFactory factory;
@Getter
private final StoreFeatures features;
private final String prefix;
private final String prefixAndMutateMany;
private final String prefixAndMutateManyUpdateOrDeleteItemCalls;
private final String prefixAndMutateManyKeys;
private final String prefixAndMutateManyStores;
private final Duration lockExpiryTime;
private static int getPort(final Configuration config) throws BackendException {
final String endpoint = JanusGraphConfigUtil.getNullableConfigValue(config, Constants.DYNAMODB_CLIENT_ENDPOINT);
int port = DEFAULT_PORT;
if (endpoint != null && !endpoint.equals(Constants.DYNAMODB_CLIENT_ENDPOINT.getDefaultValue())) {
final URL url;
try {
url = new URL(endpoint);
} catch (IOException e) {
throw new PermanentBackendException("Unable to determine port from endpoint: " + endpoint);
}
port = url.getPort();
}
return port;
}
public DynamoDBStoreManager(final Configuration backendConfig) throws BackendException {
super(backendConfig, getPort(backendConfig));
try {
client = new Client(backendConfig);
} catch (IllegalArgumentException e) {
throw new PermanentBackendException("Bad configuration used: " + backendConfig.toString(), e);
}
prefix = client.getPrefix();
factory = new TableNameDynamoDbStoreFactory();
features = initializeFeatures(backendConfig);
prefixAndMutateMany = String.format("%s_mutateMany", prefix);
prefixAndMutateManyUpdateOrDeleteItemCalls = String.format("%s_mutateManyUpdateOrDeleteItemCalls", prefix);
prefixAndMutateManyKeys = String.format("%s_mutateManyKeys", prefix);
prefixAndMutateManyStores = String.format("%s_mutateManyStores", prefix);
lockExpiryTime = backendConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE);
}
@Override
public StoreTransaction beginTransaction(final BaseTransactionConfig config) throws BackendException {
final DynamoDbStoreTransaction txh = new DynamoDbStoreTransaction(config);
return txh;
}
@Override
public void clearStorage() throws BackendException {
log.debug("Entering clearStorage");
for (AwsStore store : factory.getAllStores()) {
store.deleteStore();
}
log.debug("Exiting clearStorage returning:void");
}
@Override
public boolean exists() throws BackendException {
return client.getDelegate().listTables(new ListTablesRequest()) != null;
}
@Override
public void close() throws BackendException {
log.debug("Entering close");
for (AwsStore store : factory.getAllStores()) {
store.close();
}
client.getDelegate().shutdown();
log.debug("Exiting close returning:void");
}
@Override
public String getName() {
log.debug("Entering getName");
final String name = getClass().getSimpleName() + prefix;
log.debug("Exiting getName returning:{}", name);
return name;
}
private StandardStoreFeatures initializeFeatures(final Configuration config) {
final Builder builder = new StandardStoreFeatures.Builder();
return builder.batchMutation(true)
.cellTTL(false)
.distributed(true)
.keyConsistent(config)
.keyOrdered(false)
.localKeyPartition(false)
.locking(config.get(Constants.DYNAMODB_USE_NATIVE_LOCKING))
.multiQuery(true)
.orderedScan(false)
.preferredTimestamps(TimestampProviders.MILLI) //ignored because timestamps is false
.storeTTL(false)
.timestamps(false)
.transactional(false)
.supportsInterruption(false)
.optimisticLocking(true)
.unorderedScan(true)
.visibility(false).build();
}
@Override
public void mutateMany(final Map> mutations, final StoreTransaction txh) throws BackendException {
//this method can be called by janusgraph-core, which is not aware of our backend implementation.
//that means the keys of mutations map are the logical store names.
final Timer.Context ctxt = client.getDelegate().getTimerContext(this.prefixAndMutateMany, null /*tableName*/);
try {
final DynamoDbStoreTransaction tx = DynamoDbStoreTransaction.getTx(txh);
final List mutationWorkers = Lists.newLinkedList();
// one pass to create tasks
long updateOrDeleteItemCalls = 0;
long keys = 0;
for (Map.Entry> mutationMapEntry : mutations.entrySet()) {
final AwsStore store = openDatabase(mutationMapEntry.getKey());
final Map storeMutations = mutationMapEntry.getValue();
keys += storeMutations.size();
final Collection storeWorkers = store.createMutationWorkers(storeMutations, tx);
updateOrDeleteItemCalls += storeWorkers.size();
mutationWorkers.addAll(storeWorkers);
}
// shuffle the list of MutationWorkers so writes to edgestore and graphindex happen in parallel
Collections.shuffle(mutationWorkers);
client.getDelegate().getMeter(client.getDelegate().getMeterName(this.prefixAndMutateManyKeys, null /*tableName*/))
.mark(keys);
client.getDelegate().getMeter(client.getDelegate().getMeterName(this.prefixAndMutateManyUpdateOrDeleteItemCalls, null /*tableName*/))
.mark(updateOrDeleteItemCalls);
client.getDelegate().getMeter(client.getDelegate().getMeterName(this.prefixAndMutateManyStores, null /*tableName*/))
.mark(mutations.size());
client.getDelegate().parallelMutate(mutationWorkers);
} finally {
ctxt.stop();
}
}
@Override
public AwsStore openDatabase(@NonNull final String name) throws BackendException {
Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "database name may not be null or empty");
return factory.create(this /*manager*/, prefix, name);
}
@Override
public List getLocalKeyPartition() throws BackendException {
throw new UnsupportedOperationException();
}
@Override
public Deployment getDeployment() {
if (client.getDelegate().isEmbedded()) {
return Deployment.EMBEDDED;
}
return Deployment.REMOTE;
}
@Override
public KeyColumnValueStore openDatabase(final String name, final Container arg1) throws BackendException {
// TODO revisit for TTL
// https://github.com/awslabs/dynamodb-titan-storage-backend/issues/70
return factory.create(this /*manager*/, prefix, name);
}
public Duration getLockExpiresDuration() {
return lockExpiryTime;
}
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/DynamoDbDelegate.java
================================================
/*
* Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import java.net.SocketException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.diskstorage.PermanentBackendException;
import org.janusgraph.diskstorage.StaticBuffer;
import org.janusgraph.diskstorage.TemporaryBackendException;
import org.janusgraph.diskstorage.configuration.Configuration;
import org.janusgraph.diskstorage.locking.PermanentLockingException;
import org.janusgraph.util.stats.MetricManager;
import com.amazon.janusgraph.diskstorage.dynamodb.ExponentialBackoff.Scan;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.ParallelScanner;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.ScanSegmentWorker;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.MutateWorker;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.AmazonWebServiceRequest;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate;
import com.amazonaws.services.dynamodbv2.model.BatchWriteItemRequest;
import com.amazonaws.services.dynamodbv2.model.BatchWriteItemResult;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import com.amazonaws.services.dynamodbv2.model.ConsumedCapacity;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.CreateTableResult;
import com.amazonaws.services.dynamodbv2.model.DeleteItemRequest;
import com.amazonaws.services.dynamodbv2.model.DeleteItemResult;
import com.amazonaws.services.dynamodbv2.model.DeleteTableRequest;
import com.amazonaws.services.dynamodbv2.model.DeleteTableResult;
import com.amazonaws.services.dynamodbv2.model.DescribeTableRequest;
import com.amazonaws.services.dynamodbv2.model.DescribeTableResult;
import com.amazonaws.services.dynamodbv2.model.GetItemRequest;
import com.amazonaws.services.dynamodbv2.model.GetItemResult;
import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndexDescription;
import com.amazonaws.services.dynamodbv2.model.ListTablesRequest;
import com.amazonaws.services.dynamodbv2.model.ListTablesResult;
import com.amazonaws.services.dynamodbv2.model.LocalSecondaryIndexDescription;
import com.amazonaws.services.dynamodbv2.model.PutItemRequest;
import com.amazonaws.services.dynamodbv2.model.PutItemResult;
import com.amazonaws.services.dynamodbv2.model.QueryRequest;
import com.amazonaws.services.dynamodbv2.model.QueryResult;
import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
import com.amazonaws.services.dynamodbv2.model.ScanRequest;
import com.amazonaws.services.dynamodbv2.model.ScanResult;
import com.amazonaws.services.dynamodbv2.model.TableDescription;
import com.amazonaws.services.dynamodbv2.model.TableStatus;
import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest;
import com.amazonaws.services.dynamodbv2.model.UpdateItemResult;
import com.amazonaws.services.dynamodbv2.model.WriteRequest;
import com.amazonaws.util.AwsHostNameUtils;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.RateLimiter;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* A wrapper on top of the DynamoDB client API that self-throttles using metric-based and context-aware
* estimates for all Read APIs and for DeleteItem, and that self-throttles using accurate upper-bound
* estimates of item size for PutItem and UpdateItem. Has a thread pool and is able to do parallel
* UpdateItem / DeleteItem, Query, and Scan calls.
*
* @author Alexander Patrikalakis
*
*/
@Slf4j
public class DynamoDbDelegate {
private static final String PAGES = "Pages";
private static final String CREATE_TABLE = "CreateTable";
private static final String DELETE_TABLE = "DeleteTable";
private static final String MUTATE_ITEM = "MutateItem";
private static final String HASH_RANGE_KEY_SIZE_LIMIT = "Hash primary key values must be under 2048 bytes, and range primary key values must be under 1024 bytes";
private static final String UPDATE_ITEM_SIZE_LIMIT = "Item size to update has exceeded the maximum allowed size";
private static final String USER_AGENT = "x-amz-user-agent";
private static final String PUT_ITEM = "PutItem";
private static final String BATCH_WRITE_ITEM = "BatchWriteItem";
private static final String DESCRIBE_TABLE = "DescribeTable";
static final String UPDATE_ITEM = "UpdateItem";
static final String DELETE_ITEM = "DeleteItem";
static final String QUERY = "Query";
static final String GET_ITEM = "GetItem";
public static final String SCAN = "Scan";
private static final Charset UTF8 = Charset.forName("UTF8");
// Each List element has 1 byte overhead for type. Adding 1 byte to account for it in item size
private static final int BASE_LOGICAL_SIZE_OF_NESTED_TYPES = 1;
private static final int LOGICAL_SIZE_OF_EMPTY_DOCUMENT = 3;
private static final int MAX_NUMBER_OF_BYTES_FOR_NUMBER = 21;
private static final int ONE_KILOBYTE = 1024;
private static final long CONTROL_PLANE_RETRY_DELAY_MS = 1000;
private static final String LIST_TABLES = "ListTables";
public static final int BATCH_WRITE_MAX_NUMBER_OF_ITEMS = 25;
private final AmazonDynamoDB client;
private final ThreadPoolExecutor clientThreadPool;
private final Map readRateLimit;
private final Map writeRateLimit;
private final RateLimiter controlPlaneRateLimiter;
private final int maxConcurrentUsers;
@Getter
private final long maxRetries;
@Getter
private final long retryMillis;
@Getter
private final boolean embedded = false;
@Getter
private final String listTablesApiName;
private final String executorGaugeName;
private final String metricsPrefix;
public DynamoDbDelegate(final String endpoint, final String region, final AWSCredentialsProvider provider,
final ClientConfiguration clientConfig, final Configuration titanConfig,
final Map readRateLimit, final Map writeRateLimit,
final long maxRetries, final long retryMillis, final String prefix, final String metricsPrefix,
final RateLimiter controlPlaneRateLimiter) {
if (prefix == null) {
throw new IllegalArgumentException("prefix must be set");
}
if (metricsPrefix == null || metricsPrefix.isEmpty()) {
throw new IllegalArgumentException("metrics-prefix may not be null or empty");
}
this.metricsPrefix = metricsPrefix;
executorGaugeName = String.format("%s.%s_executor-queue-size", this.metricsPrefix, prefix);
clientThreadPool = getPoolFromNs(titanConfig);
if (!MetricManager.INSTANCE.getRegistry().getNames().contains(executorGaugeName)) {
MetricManager.INSTANCE.getRegistry().register(executorGaugeName, (Gauge) () -> clientThreadPool.getQueue().size());
}
client = AmazonDynamoDBClientBuilder.standard()
.withCredentials(provider)
.withClientConfiguration(clientConfig)
.withEndpointConfiguration(getEndpointConfiguration(Optional.ofNullable(endpoint), region))
.build();
this.readRateLimit = readRateLimit;
this.writeRateLimit = writeRateLimit;
this.controlPlaneRateLimiter = controlPlaneRateLimiter;
this.maxConcurrentUsers = titanConfig.get(Constants.DYNAMODB_CLIENT_EXECUTOR_MAX_CONCURRENT_OPERATIONS);
this.maxRetries = maxRetries;
this.retryMillis = retryMillis;
if (maxConcurrentUsers < 1) {
throw new IllegalArgumentException("need at least one user otherwise wont make progress on scan");
}
this.listTablesApiName = String.format("%s_ListTables", prefix);
}
static ThreadPoolExecutor getPoolFromNs(final Configuration ns) {
final int maxQueueSize = ns.get(Constants.DYNAMODB_CLIENT_EXECUTOR_QUEUE_MAX_LENGTH);
final ThreadFactory factory = new ThreadFactoryBuilder().setNameFormat("getDelegate-%d").build();
//begin adaptation of constructor at
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L104
final int maxPoolSize = ns.get(Constants.DYNAMODB_CLIENT_EXECUTOR_MAX_POOL_SIZE);
final int corePoolSize = ns.get(Constants.DYNAMODB_CLIENT_EXECUTOR_CORE_POOL_SIZE);
final long keepAlive = ns.get(Constants.DYNAMODB_CLIENT_EXECUTOR_KEEP_ALIVE);
final ThreadPoolExecutor executor = new ThreadPoolExecutor(corePoolSize, maxPoolSize, keepAlive,
TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(maxQueueSize), factory, new ThreadPoolExecutor.CallerRunsPolicy());
//end adaptation of constructor at
//https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L104
executor.allowCoreThreadTimeOut(false);
executor.prestartAllCoreThreads();
return executor;
}
@VisibleForTesting
static AwsClientBuilder.EndpointConfiguration getEndpointConfiguration(final Optional endpoint, final String signingRegion) {
Preconditions.checkArgument(endpoint != null, "must provide an optional endpoint and not null");
Preconditions.checkArgument(!Strings.isNullOrEmpty(signingRegion), "must provide a signing region");
final String expectedServiceEndpoint = "https://" + Region.getRegion(Regions.fromName(signingRegion)).getServiceEndpoint(AmazonDynamoDB.ENDPOINT_PREFIX);
if (endpoint.isPresent() && !Strings.isNullOrEmpty(endpoint.get())) {
final String regionParsedFromEndpoint = AwsHostNameUtils.parseRegion(endpoint.get(), AmazonDynamoDB.ENDPOINT_PREFIX);
Preconditions.checkArgument(regionParsedFromEndpoint == null || signingRegion.equals(regionParsedFromEndpoint));
return new AwsClientBuilder.EndpointConfiguration(endpoint.get(), signingRegion);
} else {
//Regions.fromName will throw IllegalArgumentException if signingRegion is not valid.
return new AwsClientBuilder.EndpointConfiguration(expectedServiceEndpoint, signingRegion);
}
}
private T setUserAgent(final T request) {
request.putCustomRequestHeader(USER_AGENT, Constants.JANUSGRAPH_USER_AGENT);
return request;
}
private BackendException processDynamoDbApiException(final Throwable e, final String apiName, final String tableName) {
Preconditions.checkArgument(apiName != null);
Preconditions.checkArgument(!apiName.isEmpty());
final String prefix;
if (tableName == null) {
prefix = apiName;
} else {
prefix = String.format("%s_%s", apiName, tableName);
}
final String message = String.format("%s %s", prefix, e.getMessage());
if (e instanceof ResourceNotFoundException) {
return new BackendNotFoundException(String.format("%s; table not found", message), e);
} else if (e instanceof ConditionalCheckFailedException) {
return new PermanentLockingException(message, e);
} else if (e instanceof AmazonServiceException) {
if (e.getMessage() != null
&& (e.getMessage().contains(HASH_RANGE_KEY_SIZE_LIMIT) || e.getMessage().contains(UPDATE_ITEM_SIZE_LIMIT))) {
return new PermanentBackendException(message, e);
} else {
return new TemporaryBackendException(message, e);
}
} else if (e instanceof AmazonClientException) { //all client exceptions are retriable by default
return new TemporaryBackendException(message, e);
} else if (e instanceof SocketException) { //sometimes this doesn't get caught by SDK
return new TemporaryBackendException(message, e);
}
// unknown exception type
return new PermanentBackendException(message, e);
}
public ScanResult scan(final ScanRequest request, final int permitsToConsume) throws BackendException {
setUserAgent(request);
ScanResult result;
timedReadThrottle(SCAN, request.getTableName(), permitsToConsume);
final Timer.Context apiTimerContext = getTimerContext(SCAN, request.getTableName());
try {
result = client.scan(request);
} catch (Exception e) {
throw processDynamoDbApiException(e, SCAN, request.getTableName());
} finally {
apiTimerContext.stop();
}
meterConsumedCapacity(SCAN, result.getConsumedCapacity());
measureItemCount(SCAN, request.getTableName(), result.getCount());
return result;
}
ParallelScanner getParallelScanCompletionService(final ScanRequest initialRequest) throws BackendException {
final int segments = Math.max(1, clientThreadPool.getMaximumPoolSize() / maxConcurrentUsers);
final ParallelScanner completion = new ParallelScanner(clientThreadPool, segments, this);
for (int segment = 0; segment < segments; segment++) {
// dont need to set user agent here because ExponentialBackoff.Scan
// calls DynamoDbDelegate.scan which sets it
final ScanRequest scanSegment = copyScanRequest(initialRequest).withTotalSegments(segments).withSegment(segment);
completion.addWorker(new ScanSegmentWorker(this, scanSegment), segment);
}
return completion;
}
public Future scanAsync(final ScanRequest request, final int permitsToConsume) {
return clientThreadPool.submit(() -> {
final Scan backoff = new Scan(request, this, permitsToConsume);
return backoff.runWithBackoff();
});
}
public static ScanRequest copyScanRequest(final ScanRequest request) {
return new ScanRequest().withAttributesToGet(request.getAttributesToGet())
.withScanFilter(request.getScanFilter())
.withConditionalOperator(request.getConditionalOperator())
.withExclusiveStartKey(request.getExclusiveStartKey())
.withExpressionAttributeNames(request.getExpressionAttributeNames())
.withExpressionAttributeValues(cloneItem(request.getExpressionAttributeValues()))
.withFilterExpression(request.getFilterExpression())
.withIndexName(request.getIndexName()).withLimit(request.getLimit())
.withProjectionExpression(request.getProjectionExpression())
.withReturnConsumedCapacity(request.getReturnConsumedCapacity())
.withScanFilter(request.getScanFilter()).withSelect(request.getSelect())
.withTableName(request.getTableName()).withTotalSegments(request.getTotalSegments())
.withSegment(request.getSegment());
}
void parallelMutate(final List workers) throws BackendException {
final CompletionService completion = new ExecutorCompletionService<>(clientThreadPool);
final List> futures = Lists.newLinkedList();
for (MutateWorker worker : workers) {
futures.add(completion.submit(worker));
}
//block on the futures all getting or throwing instead of using a latch as i need to check future status anyway
boolean interrupted = false;
try {
for (int i = 0; i < workers.size(); i++) {
try {
completion.take().get(); //Void
} catch (InterruptedException e) {
interrupted = true;
// fail out because janusgraph does not poll this thread for interrupted anywhere
throw new BackendRuntimeException("was interrupted during parallelMutate");
} catch (ExecutionException e) {
throw unwrapExecutionException(e, MUTATE_ITEM);
}
}
} finally {
for (Future future : futures) {
if (!future.isDone()) {
future.cancel(interrupted /* mayInterruptIfRunning */);
}
}
if (interrupted) {
// set interrupted on this thread
Thread.currentThread().interrupt();
}
}
}
List parallelQuery(final List queryWorkers) throws BackendException {
final CompletionService completionService = new ExecutorCompletionService<>(clientThreadPool);
final List> futures = Lists.newLinkedList();
for (QueryWorker worker : queryWorkers) {
futures.add(completionService.submit(worker));
}
boolean interrupted = false;
final List results = Lists.newLinkedList();
try {
for (int i = 0; i < queryWorkers.size(); i++) {
try {
final QueryResultWrapper result = completionService.take().get();
results.add(result);
} catch (InterruptedException e) {
interrupted = true;
// fail out because janusgraph does not poll this thread for interrupted anywhere
throw new BackendRuntimeException("was interrupted during parallelQuery");
} catch (ExecutionException e) {
throw unwrapExecutionException(e, QUERY);
}
}
} finally {
for (Future future : futures) {
if (!future.isDone()) {
future.cancel(interrupted /* mayInterruptIfRunning */);
}
}
if (interrupted) {
// set interrupted on this thread and fail out
Thread.currentThread().interrupt();
}
}
return results;
}
Map parallelGetItem(final List workers) throws BackendException {
final CompletionService completionService = new ExecutorCompletionService<>(clientThreadPool);
final List> futures = Lists.newLinkedList();
for (GetItemWorker worker : workers) {
futures.add(completionService.submit(worker));
}
boolean interrupted = false;
final Map results = Maps.newHashMap();
try {
for (int i = 0; i < workers.size(); i++) {
try {
final GetItemResultWrapper result = completionService.take().get();
results.put(result.getJanusGraphKey(), result.getDynamoDBResult());
} catch (InterruptedException e) {
interrupted = true;
throw new BackendRuntimeException("was interrupted during parallelGet");
} catch (ExecutionException e) {
throw unwrapExecutionException(e, GET_ITEM);
}
}
} finally {
for (Future future : futures) {
if (!future.isDone()) {
future.cancel(interrupted /* mayInterruptIfRunning */);
}
}
if (interrupted) {
// set interrupted on this thread and fail out
Thread.currentThread().interrupt();
}
}
return results;
}
public BackendException unwrapExecutionException(final ExecutionException e, final String apiName) {
final Throwable cause = e.getCause();
if (cause instanceof BackendException) {
return (BackendException) cause; //already translated
} else {
//ok not to drill down to specific because would have thrown permanentbackend exception for other
return processDynamoDbApiException(cause, apiName, null /*tableName*/);
}
}
GetItemResult getItem(final GetItemRequest request) throws BackendException {
setUserAgent(request);
GetItemResult result;
timedReadThrottle(GET_ITEM, request.getTableName(), estimateCapacityUnits(GET_ITEM, request.getTableName()));
final Timer.Context apiTimerContext = getTimerContext(GET_ITEM, request.getTableName());
try {
result = client.getItem(request);
} catch (Exception e) {
throw processDynamoDbApiException(e, GET_ITEM, request.getTableName());
} finally {
apiTimerContext.stop();
}
meterConsumedCapacity(GET_ITEM, result.getConsumedCapacity());
return result;
}
public BatchWriteItemResult batchWriteItem(final BatchWriteItemRequest batchRequest) throws BackendException {
int count = 0;
for (Entry> entry : batchRequest.getRequestItems().entrySet()) {
final String tableName = entry.getKey();
final List requests = entry.getValue();
count += requests.size();
if (count > BATCH_WRITE_MAX_NUMBER_OF_ITEMS) {
throw new IllegalArgumentException("cant have more than 25 requests in a batchwrite");
}
for (final WriteRequest request : requests) {
if ((request.getPutRequest() != null) == (request.getDeleteRequest() != null)) {
throw new IllegalArgumentException("Exactly one of PutRequest or DeleteRequest must be set in each WriteRequest in a batch write operation");
}
final int wcu;
final String apiName;
if (request.getPutRequest() != null) {
apiName = PUT_ITEM;
final int bytes = calculateItemSizeInBytes(request.getPutRequest().getItem());
wcu = computeWcu(bytes);
} else { //deleterequest
apiName = DELETE_ITEM;
wcu = estimateCapacityUnits(apiName, tableName);
}
timedWriteThrottle(apiName, tableName, wcu);
}
}
BatchWriteItemResult result;
setUserAgent(batchRequest);
final Timer.Context apiTimerContext = getTimerContext(BATCH_WRITE_ITEM, null /*tableName*/);
try {
result = client.batchWriteItem(batchRequest);
} catch (Exception e) {
throw processDynamoDbApiException(e, BATCH_WRITE_ITEM, null /*tableName*/);
} finally {
apiTimerContext.stop();
}
if (result.getConsumedCapacity() != null) {
for (ConsumedCapacity ccu : result.getConsumedCapacity()) {
meterConsumedCapacity(BATCH_WRITE_ITEM, ccu);
}
}
return result;
}
public QueryResult query(final QueryRequest request, final int permitsToConsume) throws BackendException {
setUserAgent(request);
QueryResult result;
timedReadThrottle(QUERY, request.getTableName(), permitsToConsume);
final Timer.Context apiTimerContext = getTimerContext(QUERY, request.getTableName());
try {
result = client.query(request);
} catch (Exception e) {
throw processDynamoDbApiException(e, QUERY, request.getTableName());
} finally {
apiTimerContext.stop();
}
meterConsumedCapacity(QUERY, result.getConsumedCapacity());
measureItemCount(QUERY, request.getTableName(), result.getCount());
return result;
}
public PutItemResult putItem(final PutItemRequest request) throws BackendException {
setUserAgent(request);
PutItemResult result;
final int bytes = calculateItemSizeInBytes(request.getItem());
getBytesHistogram(PUT_ITEM, request.getTableName()).update(bytes);
final int wcu = computeWcu(bytes);
timedWriteThrottle(PUT_ITEM, request.getTableName(), wcu);
final Timer.Context apiTimerContext = getTimerContext(PUT_ITEM, request.getTableName());
try {
result = client.putItem(request);
} catch (Exception e) {
throw processDynamoDbApiException(e, PUT_ITEM, request.getTableName());
} finally {
apiTimerContext.stop();
}
meterConsumedCapacity(PUT_ITEM, result.getConsumedCapacity());
return result;
}
UpdateItemResult updateItem(final UpdateItemRequest request) throws BackendException {
setUserAgent(request);
UpdateItemResult result;
final int bytes;
if (request.getUpdateExpression() != null) {
bytes = calculateExpressionBasedUpdateSize(request);
} else {
bytes = calculateItemUpdateSizeInBytes(request.getAttributeUpdates());
}
getBytesHistogram(UPDATE_ITEM, request.getTableName()).update(bytes);
final int wcu = computeWcu(bytes);
timedWriteThrottle(UPDATE_ITEM, request.getTableName(), wcu);
final Timer.Context apiTimerContext = getTimerContext(UPDATE_ITEM, request.getTableName());
try {
result = client.updateItem(request);
} catch (Exception e) {
throw processDynamoDbApiException(e, UPDATE_ITEM, request.getTableName());
} finally {
apiTimerContext.stop();
}
meterConsumedCapacity(UPDATE_ITEM, result.getConsumedCapacity());
return result;
}
/**
* This method calculates a lower bound of the size of a new item created with UpdateItem UpdateExpression. It does not
* account for the size of the attribute names of the document paths in the attribute names map and it assumes that the
* UpdateExpression only uses the SET action to assign to top-level attributes.
* @param request UpdateItem request that uses update expressions
* @return the size of the post-update image of the item
*/
private int calculateExpressionBasedUpdateSize(final UpdateItemRequest request) {
if (request == null || request.getUpdateExpression() == null) {
throw new IllegalArgumentException("request did not use update expression");
}
int size = calculateItemSizeInBytes(request.getKey());
for (AttributeValue value : request.getExpressionAttributeValues().values()) {
size += calculateAttributeSizeInBytes(value);
}
return size;
}
DeleteItemResult deleteItem(final DeleteItemRequest request) throws BackendException {
setUserAgent(request);
DeleteItemResult result;
final int wcu = estimateCapacityUnits(DELETE_ITEM, request.getTableName());
timedWriteThrottle(DELETE_ITEM, request.getTableName(), wcu);
final Timer.Context apiTimerContext = getTimerContext(DELETE_ITEM, request.getTableName());
try {
result = client.deleteItem(request);
} catch (Exception e) {
throw processDynamoDbApiException(e, DELETE_ITEM, request.getTableName());
} finally {
apiTimerContext.stop();
}
meterConsumedCapacity(DELETE_ITEM, result.getConsumedCapacity());
return result;
}
public int estimateCapacityUnits(final String apiName, final String tableName) {
int cu = 1;
final Meter apiCcuMeter = getConsumedCapacityMeter(apiName, tableName);
final Timer apiTimer = getTimer(apiName, tableName);
if (apiCcuMeter != null && apiTimer != null && apiTimer.getCount() > 0) {
cu = (int) Math.round(Math.max(1.0, (double) apiCcuMeter.getCount() / (double) apiTimer.getCount()));
}
return cu;
}
private RateLimiter readRateLimit(final String tableName) {
return readRateLimit.get(tableName);
}
private RateLimiter writeRateLimit(final String tableName) {
return writeRateLimit.get(tableName);
}
private void timedWriteThrottle(final String apiName, final String tableName, final int permits) {
timedThrottle(apiName, writeRateLimit(tableName), tableName, permits);
}
private void timedReadThrottle(final String apiName, final String tableName, final int permits) {
timedThrottle(apiName, readRateLimit(tableName), tableName, permits);
}
private void timedThrottle(final String apiName, final RateLimiter limiter, final String tableName, final int permits) {
if (limiter == null) {
throw new IllegalArgumentException("limiter for " + apiName + " on table " + tableName + " was null");
}
final Timer.Context throttleTimerCtxt = getTimerContext(String.format("%sThrottling", apiName), tableName);
try {
limiter.acquire(permits);
} finally {
throttleTimerCtxt.stop();
}
}
ListTablesResult listTables(final ListTablesRequest request) throws BackendException {
controlPlaneRateLimiter.acquire();
final Timer.Context apiTimerContext = getTimerContext(listTablesApiName, null /*tableName*/);
ListTablesResult result;
try {
result = client.listTables(request);
} catch (final Exception e) {
throw processDynamoDbApiException(e, LIST_TABLES, null /*tableName*/);
} finally {
apiTimerContext.stop();
}
return result;
}
public ListTablesResult listAllTables() throws BackendException {
final ListTablesWorker worker = new ListTablesWorker(this);
worker.call();
return worker.getMergedPages();
}
private TableDescription describeTable(final String tableName) throws BackendException {
return describeTable(new DescribeTableRequest().withTableName(tableName)).getTable();
}
private DescribeTableResult describeTable(final DescribeTableRequest request) throws BackendException {
controlPlaneRateLimiter.acquire();
final Timer.Context apiTimerContext = getTimerContext(DESCRIBE_TABLE, request.getTableName());
DescribeTableResult result;
try {
result = client.describeTable(request);
} catch (final Exception e) {
throw processDynamoDbApiException(e, DESCRIBE_TABLE, request.getTableName());
} finally {
apiTimerContext.stop();
}
return result;
}
public DeleteTableResult deleteTable(final DeleteTableRequest request) throws BackendException {
controlPlaneRateLimiter.acquire();
final Timer.Context apiTimerContext = getTimerContext(DELETE_TABLE, request.getTableName());
DeleteTableResult result;
try {
result = client.deleteTable(request);
} catch (Exception e) {
throw processDynamoDbApiException(e, DELETE_TABLE, request.getTableName());
} finally {
apiTimerContext.stop();
}
return result;
}
private void interruptibleSleep(final long millis) {
boolean interrupted = false;
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
interrupted = true;
} finally {
if (interrupted) {
Thread.currentThread().interrupt();
}
}
}
boolean ensureTableDeleted(final String tableName) throws BackendException {
boolean successFlag = false;
int retryCount = 0;
do {
try {
this.describeTable(tableName);
} catch (BackendNotFoundException e) {
successFlag = true;
break;
}
interruptibleSleep(CONTROL_PLANE_RETRY_DELAY_MS);
retryCount++;
} while (!successFlag && retryCount < maxRetries);
if (!successFlag) {
throw new PermanentBackendException("Table deletion not completed after retrying " + maxRetries + " times");
}
return successFlag;
}
DeleteTableResult deleteTable(final String tableName) throws BackendException {
return deleteTable(new DeleteTableRequest().withTableName(tableName));
}
private CreateTableResult createTable(final CreateTableRequest request) throws BackendException {
controlPlaneRateLimiter.acquire();
final Timer.Context apiTimerContext = getTimerContext(CREATE_TABLE, request.getTableName());
CreateTableResult result;
try {
result = client.createTable(request);
} catch (final Exception e) {
throw processDynamoDbApiException(e, CREATE_TABLE, request.getTableName());
} finally {
apiTimerContext.stop();
}
return result;
}
private static boolean isTableAcceptingWrites(final String status) {
return isTableStatus(TableStatus.ACTIVE, status) || isTableStatus(TableStatus.UPDATING, status);
}
private static boolean isTableStatus(final TableStatus constant, final String status) {
return constant.toString().equals(status);
}
public void waitForTableCreation(final String tableName, final boolean verifyIndexesList,
final List expectedLsiList, final List expectedGsiList) throws BackendException {
boolean successFlag = false;
int retryCount = 0;
while (!successFlag && retryCount < maxRetries) {
try {
boolean areAllGsisActive = true;
final TableDescription td = describeTable(tableName);
if (verifyIndexesList) {
final Set expectedLSIs = new HashSet();
if (expectedLsiList != null) {
expectedLSIs.addAll(expectedLsiList);
}
final Set actualLSIs = new HashSet();
if (td.getLocalSecondaryIndexes() != null) {
actualLSIs.addAll(td.getLocalSecondaryIndexes());
}
// the lsi list should be there even if the table is in creating state
if (!(expectedLsiList == null && td.getLocalSecondaryIndexes() == null || expectedLSIs.equals(actualLSIs))) {
throw new PermanentBackendException("LSI list is not as expected during table creation. expectedLsiList="
+ expectedLsiList.toString() + "; table description=" + td.toString());
}
// ignore the status of all GSIs since they will mess up .equals()
if (td.getGlobalSecondaryIndexes() != null) {
for (final GlobalSecondaryIndexDescription gDesc : td.getGlobalSecondaryIndexes()) {
if (!isTableAcceptingWrites(gDesc.getIndexStatus())) {
areAllGsisActive = false;
break;
}
}
}
// the gsi list should be there even if the table is in creating state
if (!areGsisSameConfiguration(expectedGsiList, td.getGlobalSecondaryIndexes())) {
throw new PermanentBackendException("GSI list is not as expected during table creation. expectedGsiList="
+ expectedGsiList.toString() + "; table description=" + td.toString());
}
}
successFlag = isTableAcceptingWrites(td.getTableStatus()) && areAllGsisActive;
} catch (BackendNotFoundException ignore) {
successFlag = false;
}
if (!successFlag) {
interruptibleSleep(CONTROL_PLANE_RETRY_DELAY_MS);
}
retryCount++;
}
if (!successFlag) {
throw new PermanentBackendException("Table creation not completed for table " + tableName + " after retrying "
+ this.maxRetries + " times for a duration of " + CONTROL_PLANE_RETRY_DELAY_MS * this.maxRetries + " ms");
}
}
private static boolean areGsisSameConfiguration(final List g1,
final List g2) {
if (g1 == null) {
return g2 == null;
}
if (g1.size() != g2.size()) {
return false;
}
// make copy of the lists because we don't want to mutate the lists
final ArrayList g1clone = new ArrayList<>(g1.size());
g1clone.addAll(g1);
final ArrayList g2clone = new ArrayList<>(g2.size());
g1clone.addAll(g2);
for (final GlobalSecondaryIndexDescription gi1 : g1) {
for (final GlobalSecondaryIndexDescription gi2 : g2) {
if (areGsisSameConfiguration(gi1, gi2)) {
g1clone.remove(gi1);
g2clone.remove(gi2);
break;
}
}
}
return g1clone.isEmpty() || g2clone.isEmpty();
}
private static boolean areGsisSameConfiguration(final GlobalSecondaryIndexDescription g1, final GlobalSecondaryIndexDescription g2) {
if (g1 == null ^ g2 == null) {
return false;
}
if (g1 == g2) {
return true;
}
final EqualsBuilder builder = new EqualsBuilder();
builder.append(g1.getIndexName(), g2.getIndexName());
builder.append(g1.getKeySchema(), g2.getKeySchema());
builder.append(g1.getProjection().getProjectionType(), g2.getProjection().getProjectionType());
builder.append(g1.getProvisionedThroughput().getReadCapacityUnits(), g2.getProvisionedThroughput().getReadCapacityUnits());
builder.append(g1.getProvisionedThroughput().getWriteCapacityUnits(), g2.getProvisionedThroughput().getWriteCapacityUnits());
final Set projectionNonKeyAttributesG1 =
new HashSet<>(Optional.ofNullable(g1.getProjection().getNonKeyAttributes()).orElse(Collections.emptyList()));
final Set projectionNonKeyAttributesG2 =
new HashSet<>(Optional.ofNullable(g2.getProjection().getNonKeyAttributes()).orElse(Collections.emptyList()));
builder.append(projectionNonKeyAttributesG1, projectionNonKeyAttributesG2);
return builder.build();
}
void createTableAndWaitForActive(final CreateTableRequest request) throws BackendException {
final String tableName = request.getTableName();
Preconditions.checkArgument(!Strings.isNullOrEmpty(tableName), "Table name was null or empty");
final TableDescription desc;
try {
desc = this.describeTable(tableName);
if (null != desc && isTableAcceptingWrites(desc.getTableStatus())) {
return; //store existed
}
} catch (BackendNotFoundException e) {
log.debug(tableName + " did not exist yet, creating it", e);
}
createTable(request);
waitForTableCreation(tableName, false /*verifyIndexesList*/, null /*expectedLsiList*/, null /*expectedGsiList*/);
}
public void shutdown() {
MetricManager.INSTANCE.getRegistry().remove(executorGaugeName);
// TODO(amcp) figure out a way to make the thread pool not be static
// https://github.com/awslabs/dynamodb-titan-storage-backend/issues/48
client.shutdown();
}
private Timer getTimer(final String apiName, final String tableName) {
return MetricManager.INSTANCE.getTimer(getMeterName(apiName, tableName));
}
final Timer.Context getTimerContext(final String apiName, final String tableName) {
return getTimer(apiName, tableName).time();
}
final Meter getMeter(final String meterName) {
return MetricManager.INSTANCE.getRegistry().meter(meterName);
}
private String getItemCountMeterName(final String apiName, final String tableName) {
return getMeterName(String.format("%sItemCount", apiName), tableName);
}
private void measureItemCount(final String apiName, final String tableName, final long itemCount) {
getMeter(getItemCountMeterName(apiName, tableName)).mark(itemCount);
getCounter(apiName, tableName, "ItemCountCounter").inc(itemCount);
}
private Counter getCounter(final String apiName, final String tableName, final String quantity) {
return MetricManager.INSTANCE.getCounter(getQuantityName(apiName, tableName, quantity));
}
private void meterConsumedCapacity(final String apiName, final ConsumedCapacity ccu) {
if (ccu != null) {
getConsumedCapacityMeter(apiName, ccu.getTableName()).mark(Math.round(ccu.getCapacityUnits()));
}
}
private String getQuantityName(final String apiName, final String tableName, final String quantity) {
return getMeterName(String.format("%s%s", apiName, quantity), tableName);
}
private Meter getQuantityMeter(final String apiName, final String tableName, final String quantity) {
return getMeter(getQuantityName(apiName, tableName, quantity));
}
private Meter getConsumedCapacityMeter(final String apiName, final String tableName) {
return getQuantityMeter(apiName, tableName, "ConsumedCapacity");
}
private Histogram getBytesHistogram(final String apiName, final String tableName) {
return getHistogram(apiName, tableName, "Bytes");
}
private Histogram getHistogram(final String apiName, final String tableName, final String quantity) {
return MetricManager.INSTANCE.getHistogram(getQuantityName(apiName, tableName, quantity));
}
public final Histogram getPagesHistogram(final String apiName, final String tableName) {
return getHistogram(apiName, tableName, PAGES);
}
void updatePagesHistogram(final String apiName, final String tableName, final int pagesProcessed) {
getHistogram(apiName, tableName, PAGES).update(pagesProcessed);
}
final String getMeterName(final String apiName, final String tableName) {
if (tableName == null) {
return String.format("%s.%s", metricsPrefix, apiName);
}
return String.format("%s.%s.%s", metricsPrefix, apiName, tableName);
}
final int getMaxConcurrentUsers() {
return this.maxConcurrentUsers;
}
/**
* Helper method that clones an item
*
* @param item the item to clone
* @return a clone of item.
*/
public static Map cloneItem(final Map item) {
if (item == null) {
return null;
}
final Map clonedItem = Maps.newHashMap();
final IdentityHashMap sourceDestinationMap = new IdentityHashMap<>();
for (Entry entry : item.entrySet()) {
if (!sourceDestinationMap.containsKey(entry.getValue())) {
sourceDestinationMap.put(entry.getValue(), clone(entry.getValue(), sourceDestinationMap));
}
clonedItem.put(entry.getKey(), sourceDestinationMap.get(entry.getValue()));
}
return clonedItem;
}
/**
* Helper method that can clone an Attribute Value
*
* @param val the AttributeValue to copy
* @param sourceDestinationMap used to avoid loops by keeping track of references
* @return a copy of val
*/
public static AttributeValue clone(final AttributeValue val, final IdentityHashMap sourceDestinationMap) {
if (val == null) {
return null;
}
if (sourceDestinationMap.containsKey(val)) {
return sourceDestinationMap.get(val);
}
final AttributeValue clonedVal = new AttributeValue();
sourceDestinationMap.put(val, clonedVal);
if (val.getN() != null) {
clonedVal.setN(val.getN());
} else if (val.getS() != null) {
clonedVal.setS(val.getS());
} else if (val.getB() != null) {
clonedVal.setB(val.getB());
} else if (val.getNS() != null) {
clonedVal.setNS(val.getNS());
} else if (val.getSS() != null) {
clonedVal.setSS(val.getSS());
} else if (val.getBS() != null) {
clonedVal.setBS(val.getBS());
} else if (val.getBOOL() != null) {
clonedVal.setBOOL(val.getBOOL());
} else if (val.getNULL() != null) {
clonedVal.setNULL(val.getNULL());
} else if (val.getL() != null) {
final List list = new ArrayList<>(val.getL().size());
for (AttributeValue listItemValue : val.getL()) {
if (!sourceDestinationMap.containsKey(listItemValue)) {
sourceDestinationMap.put(listItemValue, clone(listItemValue, sourceDestinationMap));
}
list.add(sourceDestinationMap.get(listItemValue));
}
clonedVal.setL(list);
} else if (val.getM() != null) {
final Map map = new HashMap<>(val.getM().size());
for (Entry pair : val.getM().entrySet()) {
if (!sourceDestinationMap.containsKey(pair.getValue())) {
sourceDestinationMap.put(pair.getValue(), clone(pair.getValue(), sourceDestinationMap));
}
map.put(pair.getKey(), sourceDestinationMap.get(pair.getValue()));
}
clonedVal.setM(map);
}
return clonedVal;
}
public static final int computeWcu(final int bytes) {
return Math.max(1, Integer.divideUnsigned(bytes, ONE_KILOBYTE));
}
/**Calculate attribute value size*/
private static int calculateAttributeSizeInBytes(final AttributeValue value) {
int attrValSize = 0;
if (value == null) {
return attrValSize;
}
if (value.getB() != null) {
final ByteBuffer b = value.getB();
attrValSize += b.remaining();
} else if (value.getS() != null) {
final String s = value.getS();
attrValSize += s.getBytes(UTF8).length;
} else if (value.getN() != null) {
attrValSize += MAX_NUMBER_OF_BYTES_FOR_NUMBER;
} else if (value.getBS() != null) {
final List bs = value.getBS();
for (ByteBuffer b : bs) {
if (b != null) {
attrValSize += b.remaining();
}
}
} else if (value.getSS() != null) {
final List ss = value.getSS();
for (String s : ss) {
if (s != null) {
attrValSize += s.getBytes(UTF8).length;
}
}
} else if (value.getNS() != null) {
final List ns = value.getNS();
for (String n : ns) {
if (n != null) {
attrValSize += MAX_NUMBER_OF_BYTES_FOR_NUMBER;
}
}
} else if (value.getBOOL() != null) {
attrValSize += 1;
} else if (value.getNULL() != null) {
attrValSize += 1;
} else if (value.getM() != null) {
for (Map.Entry entry : value.getM().entrySet()) {
attrValSize += entry.getKey().getBytes(UTF8).length;
attrValSize += calculateAttributeSizeInBytes(entry.getValue());
attrValSize += BASE_LOGICAL_SIZE_OF_NESTED_TYPES;
}
attrValSize += LOGICAL_SIZE_OF_EMPTY_DOCUMENT;
} else if (value.getL() != null) {
final List list = value.getL();
for (Integer i = 0; i < list.size(); i++) {
attrValSize += calculateAttributeSizeInBytes(list.get(i));
attrValSize += BASE_LOGICAL_SIZE_OF_NESTED_TYPES;
}
attrValSize += LOGICAL_SIZE_OF_EMPTY_DOCUMENT;
}
return attrValSize;
}
public static int calculateItemUpdateSizeInBytes(final Map item) {
int size = 0;
if (item == null) {
return size;
}
for (Map.Entry entry : item.entrySet()) {
final String name = entry.getKey();
final AttributeValueUpdate update = entry.getValue();
size += name.getBytes(UTF8).length;
size += calculateAttributeSizeInBytes(update.getValue());
}
return size;
}
public static int calculateItemSizeInBytes(final Map item) {
int size = 0;
if (item == null) {
return size;
}
for (Map.Entry entry : item.entrySet()) {
final String name = entry.getKey();
final AttributeValue value = entry.getValue();
size += name.getBytes(UTF8).length;
size += calculateAttributeSizeInBytes(value);
}
return size;
}
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/DynamoDbSingleRowStore.java
================================================
/*
* Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.diskstorage.Entry;
import org.janusgraph.diskstorage.EntryList;
import org.janusgraph.diskstorage.StaticBuffer;
import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation;
import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator;
import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery;
import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
import org.janusgraph.diskstorage.util.StaticArrayEntryList;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.EntryBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.ItemBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.SingleExpectedAttributeValueBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.SingleUpdateBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.ScanBackedKeyIterator;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.Scanner;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.SequentialScanner;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.SingleRowScanInterpreter;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.MutateWorker;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.SingleUpdateWithCleanupWorker;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.UpdateItemWorker;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue;
import com.amazonaws.services.dynamodbv2.model.GetItemRequest;
import com.amazonaws.services.dynamodbv2.model.GetItemResult;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.KeyType;
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import com.amazonaws.services.dynamodbv2.model.ScanRequest;
import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
/**
* Acts as if DynamoDB were a Column Oriented Database by using key as the hash
* key and each entry has their own column. Note that if you are likely to go
* over the DynamoDB 400kb per item limit you should use DynamoDbStore.
*
* See configuration
* storage.dynamodb.stores.***store_name***.data-model=SINGLE
*
* KCV Schema - actual table (Hash(S) only):
* hk | 0x02 | 0x04 <-Attribute Names
* 0x01 | 0x03 | 0x05 <-Row Values
*
* @author Matthew Sowders
* @author Alexander Patrikalakis
*
*/
@Slf4j
public class DynamoDbSingleRowStore extends AbstractDynamoDbStore {
DynamoDbSingleRowStore(final DynamoDBStoreManager manager, final String prefix, final String storeName) {
super(manager, prefix, storeName);
}
@Override
public CreateTableRequest getTableSchema() {
return super.getTableSchema()
.withAttributeDefinitions(
new AttributeDefinition()
.withAttributeName(Constants.JANUSGRAPH_HASH_KEY)
.withAttributeType(ScalarAttributeType.S))
.withKeySchema(
new KeySchemaElement()
.withAttributeName(Constants.JANUSGRAPH_HASH_KEY)
.withKeyType(KeyType.HASH));
}
@Override
public KeyIterator getKeys(final KeyRangeQuery query, final StoreTransaction txh) throws BackendException {
throw new UnsupportedOperationException("Keys are not byte ordered.");
}
private GetItemWorker createGetItemWorker(final StaticBuffer hashKey) {
final GetItemRequest request = super.createGetItemRequest().withKey(new ItemBuilder().hashKey(hashKey).build());
return new GetItemWorker(hashKey, request, client.getDelegate());
}
private EntryList extractEntriesFromGetItemResult(final GetItemResult result, final StaticBuffer sliceStart, final StaticBuffer sliceEnd, final int limit) {
final Map item = result.getItem();
List filteredEntries = Collections.emptyList();
if (null != item) {
item.remove(Constants.JANUSGRAPH_HASH_KEY);
filteredEntries = new EntryBuilder(item)
.slice(sliceStart, sliceEnd)
.limit(limit)
.buildAll();
}
return StaticArrayEntryList.of(filteredEntries);
}
@Override
public KeyIterator getKeys(final SliceQuery query, final StoreTransaction txh) throws BackendException {
log.debug("Entering getKeys table:{} query:{} txh:{}", getTableName(), encodeForLog(query), txh);
final ScanRequest scanRequest = super.createScanRequest();
final Scanner scanner;
if (client.isEnableParallelScan()) {
scanner = client.getDelegate().getParallelScanCompletionService(scanRequest);
} else {
scanner = new SequentialScanner(client.getDelegate(), scanRequest);
}
// Because SINGLE records cannot be split across scan results, we can use the same interpreter for both
// sequential and parallel scans.
final KeyIterator result = new ScanBackedKeyIterator(scanner, new SingleRowScanInterpreter(query));
log.debug("Exiting getKeys table:{} query:{} txh:{} returning:{}", getTableName(), encodeForLog(query), txh, result);
return result;
}
@Override
public EntryList getSlice(final KeySliceQuery query, final StoreTransaction txh) throws BackendException {
log.debug("Entering getSliceKeySliceQuery table:{} query:{} txh:{}", getTableName(), encodeForLog(query), txh);
final GetItemRequest request = super.createGetItemRequest().withKey(new ItemBuilder().hashKey(query.getKey()).build());
final GetItemResult result = new ExponentialBackoff.GetItem(request, client.getDelegate()).runWithBackoff();
final List filteredEntries = extractEntriesFromGetItemResult(result, query.getSliceStart(), query.getSliceEnd(), query.getLimit());
log.debug("Exiting getSliceKeySliceQuery table:{} query:{} txh:{} returning:{}", getTableName(), encodeForLog(query), txh,
filteredEntries.size());
return StaticArrayEntryList.of(filteredEntries);
}
@Override
public Map getSlice(final List keys, final SliceQuery query, final StoreTransaction txh) throws BackendException {
log.debug("Entering getSliceMultiSliceQuery table:{} keys:{} query:{} txh:{}", getTableName(), encodeForLog(keys), encodeForLog(query),
txh);
final Map entries =
//convert keys to get item workers and get the items
client.getDelegate().parallelGetItem(keys.stream().map(this::createGetItemWorker).collect(Collectors.toList()))
.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> extractEntriesFromGetItemResult(entry.getValue(),
query.getSliceStart(), query.getSliceEnd(), query.getLimit())));
log.debug("Exiting getSliceMultiSliceQuery table:{} keys:{} query:{} txh:{} returning:{}",
getTableName(),
encodeForLog(keys),
encodeForLog(query),
txh,
entries.size());
return entries;
}
@Override
public void mutate(final StaticBuffer hashKey, final List additions, final List deletions, final StoreTransaction txh) throws BackendException {
log.debug("Entering mutate table:{} keys:{} additions:{} deletions:{} txh:{}",
getTableName(),
encodeKeyForLog(hashKey),
encodeForLog(additions),
encodeForLog(deletions),
txh);
super.mutateOneKey(hashKey, new KCVMutation(additions, deletions), txh);
log.debug("Exiting mutate table:{} keys:{} additions:{} deletions:{} txh:{} returning:void",
getTableName(),
encodeKeyForLog(hashKey),
encodeForLog(additions),
encodeForLog(deletions),
txh);
}
@Override
public Collection createMutationWorkers(final Map mutationMap, final DynamoDbStoreTransaction txh) {
final List workers = Lists.newLinkedList();
for (Map.Entry entry : mutationMap.entrySet()) {
final StaticBuffer hashKey = entry.getKey();
final KCVMutation mutation = entry.getValue();
final Map key = new ItemBuilder().hashKey(hashKey)
.build();
// Using ExpectedAttributeValue map to handle large mutations in a single request
// Large mutations would require multiple requests using expressions
final Map expected =
new SingleExpectedAttributeValueBuilder(this, txh, hashKey).build(mutation);
final Map attributeValueUpdates =
new SingleUpdateBuilder().deletions(mutation.getDeletions())
.additions(mutation.getAdditions())
.build();
final UpdateItemRequest request = super.createUpdateItemRequest()
.withKey(key)
.withReturnValues(ReturnValue.ALL_NEW)
.withAttributeUpdates(attributeValueUpdates)
.withExpected(expected);
final MutateWorker worker;
if (mutation.hasDeletions() && !mutation.hasAdditions()) {
worker = new SingleUpdateWithCleanupWorker(request, client.getDelegate());
} else {
worker = new UpdateItemWorker(request, client.getDelegate());
}
workers.add(worker);
}
return workers;
}
}
================================================
FILE: src/main/java/com/amazon/janusgraph/diskstorage/dynamodb/DynamoDbStore.java
================================================
/*
* Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.janusgraph.diskstorage.dynamodb;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.diskstorage.Entry;
import org.janusgraph.diskstorage.EntryList;
import org.janusgraph.diskstorage.StaticBuffer;
import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation;
import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator;
import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery;
import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
import org.janusgraph.diskstorage.util.StaticArrayEntryList;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.ConditionExpressionBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.EntryBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.FilterExpressionBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.ItemBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.builder.MultiUpdateExpressionBuilder;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.MultiRowParallelScanInterpreter;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.MultiRowSequentialScanInterpreter;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.ScanBackedKeyIterator;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.ScanContextInterpreter;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.Scanner;
import com.amazon.janusgraph.diskstorage.dynamodb.iterator.SequentialScanner;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.DeleteItemWorker;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.MutateWorker;
import com.amazon.janusgraph.diskstorage.dynamodb.mutation.UpdateItemWorker;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.DeleteItemRequest;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.KeyType;
import com.amazonaws.services.dynamodbv2.model.QueryRequest;
import com.amazonaws.services.dynamodbv2.model.QueryResult;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import com.amazonaws.services.dynamodbv2.model.ScanRequest;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
/**
* Acts as if DynamoDB were a Column Oriented Database by using range query when
* required.
*
* See configuration
* storage.dynamodb.stores.***table_name***.data-model=MULTI
*
* KCV Schema - actual table (Hash(S) + Range(S)):
* hk(S) | rk(S) | v(B) <-Attribute Names
* 0x01 | 0x02 | 0x03 <-Row Values
* 0x01 | 0x04 | 0x05 <-Row Values
*
* @author Matthew Sowders
* @author Alexander Patrikalakis
* @author Michael Rodaitis
*
*/
@Slf4j
public class DynamoDbStore extends AbstractDynamoDbStore {
public DynamoDbStore(final DynamoDBStoreManager manager, final String prefix, final String storeName) {
super(manager, prefix, storeName);
}
private EntryList createEntryListFromItems(final List