Repository: cld378632668/YCSB_leveldb_leveldbjni_rocksdb Branch: master Commit: 951357dbafd5 Files: 384 Total size: 1.6 MB Directory structure: gitextract_7z7eehwx/ ├── .editorconfig ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── Todo.md ├── accumulo1.6/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── conf/ │ │ │ └── accumulo.properties │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── accumulo/ │ │ ├── AccumuloClient.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── accumulo/ │ │ └── AccumuloTest.java │ └── resources/ │ └── log4j.properties ├── accumulo1.7/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── conf/ │ │ │ └── accumulo.properties │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── accumulo/ │ │ ├── AccumuloClient.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── accumulo/ │ │ └── AccumuloTest.java │ └── resources/ │ └── log4j.properties ├── accumulo1.8/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── conf/ │ │ │ └── accumulo.properties │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── accumulo/ │ │ ├── AccumuloClient.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── accumulo/ │ │ └── AccumuloTest.java │ └── resources/ │ └── log4j.properties ├── aerospike/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── AerospikeClient.java │ └── package-info.java ├── arangodb/ │ ├── .gitignore │ ├── README.md │ ├── conf/ │ │ └── logback.xml │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── ArangoDBClient.java │ └── package-info.java ├── arangodb3/ │ ├── .gitignore │ ├── README.md │ ├── conf/ │ │ └── logback.xml │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── arangodb/ │ ├── ArangoDB3Client.java │ └── package-info.java ├── asynchbase/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── AsyncHBaseClient.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ ├── google/ │ │ │ └── common/ │ │ │ ├── base/ │ │ │ │ └── Stopwatch.java │ │ │ └── io/ │ │ │ ├── Closeables.java │ │ │ └── LimitInputStream.java │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── AsyncHBaseTest.java │ └── resources/ │ ├── hbase-site.xml │ └── log4j.properties ├── azuredocumentdb/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── azuredocumentdb/ │ ├── AzureDocumentDBClient.java │ └── package-info.java ├── azuretablestorage/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── azuretablestorage/ │ ├── AzureClient.java │ └── package-info.java ├── bin/ │ ├── bindings.properties │ ├── ycsb │ ├── ycsb.bat │ └── ycsb.sh ├── binding-parent/ │ ├── datastore-specific-descriptor/ │ │ ├── pom.xml │ │ └── src/ │ │ └── main/ │ │ └── resources/ │ │ └── assemblies/ │ │ └── datastore-specific-assembly.xml │ └── pom.xml ├── cassandra/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── CassandraCQLClient.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── CassandraCQLClientTest.java │ └── resources/ │ └── ycsb.cql ├── checkstyle.xml ├── cloudspanner/ │ ├── README.md │ ├── conf/ │ │ └── cloudspanner.properties │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── cloudspanner/ │ ├── CloudSpannerClient.java │ └── package-info.java ├── core/ │ ├── CHANGES.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ ├── BasicDB.java │ │ │ ├── BasicTSDB.java │ │ │ ├── ByteArrayByteIterator.java │ │ │ ├── ByteIterator.java │ │ │ ├── Client.java │ │ │ ├── CommandLine.java │ │ │ ├── DB.java │ │ │ ├── DBException.java │ │ │ ├── DBFactory.java │ │ │ ├── DBWrapper.java │ │ │ ├── GoodBadUglyDB.java │ │ │ ├── InputStreamByteIterator.java │ │ │ ├── NumericByteIterator.java │ │ │ ├── RandomByteIterator.java │ │ │ ├── Status.java │ │ │ ├── StringByteIterator.java │ │ │ ├── TerminatorThread.java │ │ │ ├── UnknownDBException.java │ │ │ ├── Utils.java │ │ │ ├── Workload.java │ │ │ ├── WorkloadException.java │ │ │ ├── generator/ │ │ │ │ ├── AcknowledgedCounterGenerator.java │ │ │ │ ├── ConstantIntegerGenerator.java │ │ │ │ ├── CounterGenerator.java │ │ │ │ ├── DiscreteGenerator.java │ │ │ │ ├── ExponentialGenerator.java │ │ │ │ ├── FileGenerator.java │ │ │ │ ├── Generator.java │ │ │ │ ├── HistogramGenerator.java │ │ │ │ ├── HotspotIntegerGenerator.java │ │ │ │ ├── IncrementingPrintableStringGenerator.java │ │ │ │ ├── NumberGenerator.java │ │ │ │ ├── RandomDiscreteTimestampGenerator.java │ │ │ │ ├── ScrambledZipfianGenerator.java │ │ │ │ ├── SequentialGenerator.java │ │ │ │ ├── SkewedLatestGenerator.java │ │ │ │ ├── UniformGenerator.java │ │ │ │ ├── UniformLongGenerator.java │ │ │ │ ├── UnixEpochTimestampGenerator.java │ │ │ │ ├── ZipfianGenerator.java │ │ │ │ └── package-info.java │ │ │ ├── measurements/ │ │ │ │ ├── Measurements.java │ │ │ │ ├── OneMeasurement.java │ │ │ │ ├── OneMeasurementHdrHistogram.java │ │ │ │ ├── OneMeasurementHistogram.java │ │ │ │ ├── OneMeasurementRaw.java │ │ │ │ ├── OneMeasurementTimeSeries.java │ │ │ │ ├── TwoInOneMeasurement.java │ │ │ │ ├── exporter/ │ │ │ │ │ ├── JSONArrayMeasurementsExporter.java │ │ │ │ │ ├── JSONMeasurementsExporter.java │ │ │ │ │ ├── MeasurementsExporter.java │ │ │ │ │ ├── TextMeasurementsExporter.java │ │ │ │ │ └── package-info.java │ │ │ │ └── package-info.java │ │ │ ├── package-info.java │ │ │ └── workloads/ │ │ │ ├── ConstantOccupancyWorkload.java │ │ │ ├── CoreWorkload.java │ │ │ ├── RestWorkload.java │ │ │ ├── TimeSeriesWorkload.java │ │ │ └── package-info.java │ │ └── resources/ │ │ └── project.properties │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ ├── TestByteIterator.java │ ├── TestNumericByteIterator.java │ ├── TestStatus.java │ ├── TestUtils.java │ ├── generator/ │ │ ├── AcknowledgedCounterGeneratorTest.java │ │ ├── TestIncrementingPrintableStringGenerator.java │ │ ├── TestRandomDiscreteTimestampGenerator.java │ │ ├── TestUnixEpochTimestampGenerator.java │ │ └── TestZipfianGenerator.java │ ├── measurements/ │ │ └── exporter/ │ │ └── TestMeasurementsExporter.java │ └── workloads/ │ ├── TestCoreWorkload.java │ └── TestTimeSeriesWorkload.java ├── couchbase/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── CouchbaseClient.java │ └── package-info.java ├── couchbase2/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── couchbase2/ │ ├── Couchbase2Client.java │ └── package-info.java ├── distribution/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── assembly/ │ └── distribution.xml ├── doc/ │ ├── coreproperties.html │ ├── coreworkloads.html │ ├── dblayer.html │ ├── index.html │ ├── parallelclients.html │ ├── tipsfaq.html │ └── workload.html ├── dynamodb/ │ ├── README.md │ ├── conf/ │ │ ├── AWSCredentials.properties │ │ └── dynamodb.properties │ ├── pom.xml │ └── src/ │ └── main/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── DynamoDBClient.java │ │ └── package-info.java │ └── resources/ │ └── log4j.properties ├── elasticsearch/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── ElasticsearchClient.java │ │ └── package-info.java │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── ElasticsearchClientTest.java ├── elasticsearch5/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ └── db/ │ │ │ └── elasticsearch5/ │ │ │ ├── Elasticsearch5.java │ │ │ ├── ElasticsearchClient.java │ │ │ ├── ElasticsearchRestClient.java │ │ │ └── package-info.java │ │ └── resources/ │ │ └── log4j2.properties │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── elasticsearch5/ │ ├── ElasticsearchClientIT.java │ ├── ElasticsearchIntegTestBase.java │ └── ElasticsearchRestClientIT.java ├── geode/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── GeodeClient.java │ └── package-info.java ├── googlebigtable/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── GoogleBigtableClient.java │ └── package-info.java ├── googledatastore/ │ ├── README.md │ ├── conf/ │ │ └── googledatastore.properties │ ├── pom.xml │ └── src/ │ └── main/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── GoogleDatastoreClient.java │ │ └── package-info.java │ └── resources/ │ └── log4j.properties ├── hbase098/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── HBaseClient.java │ └── package-info.java ├── hbase10/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── HBaseClient10.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── HBaseClient10Test.java │ └── resources/ │ ├── hbase-site.xml │ └── log4j.properties ├── hbase12/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── hbase12/ │ │ ├── HBaseClient12.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── hbase12/ │ │ └── HBaseClient12Test.java │ └── resources/ │ ├── hbase-site.xml │ └── log4j.properties ├── hypertable/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── HypertableClient.java │ └── package-info.java ├── infinispan/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── conf/ │ │ ├── infinispan-config.xml │ │ └── remote-cache.properties │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── InfinispanClient.java │ ├── InfinispanRemoteClient.java │ ├── RemoteCacheManagerHolder.java │ └── package-info.java ├── jdbc/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── conf/ │ │ │ ├── db.properties │ │ │ └── h2.properties │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ └── db/ │ │ │ ├── JdbcDBCli.java │ │ │ ├── JdbcDBClient.java │ │ │ ├── JdbcDBCreateTable.java │ │ │ ├── StatementType.java │ │ │ ├── flavors/ │ │ │ │ ├── DBFlavor.java │ │ │ │ ├── DefaultDBFlavor.java │ │ │ │ ├── PhoenixDBFlavor.java │ │ │ │ └── package-info.java │ │ │ └── package-info.java │ │ └── resources/ │ │ └── sql/ │ │ ├── README.md │ │ ├── create_table.mysql │ │ └── create_table.sql │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── JdbcDBClientTest.java ├── kudu/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── conf/ │ │ └── log4j.properties │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── KuduYCSBClient.java │ │ └── package-info.java │ └── resources/ │ └── log4j.properties ├── leveldb/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── LevelDbClient.java ├── leveldbjni/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── LevelDbJniClient.java ├── mapkeeper/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── MapKeeperClient.java ├── memcached/ │ ├── README.md │ ├── conf/ │ │ └── memcached.properties │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── MemcachedClient.java │ └── package-info.java ├── mongodb/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ └── db/ │ │ │ ├── AsyncMongoDbClient.java │ │ │ ├── MongoDbClient.java │ │ │ ├── OptionsSupport.java │ │ │ └── package-info.java │ │ └── resources/ │ │ ├── log4j.properties │ │ └── logback.xml │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── AbstractDBTestCases.java │ ├── AsyncMongoDbClientTest.java │ ├── MongoDbClientTest.java │ └── OptionsSupportTest.java ├── nosqldb/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── conf/ │ │ ├── nosqldb.properties │ │ └── script.txt │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── NoSqlDbClient.java │ └── package-info.java ├── orientdb/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ └── db/ │ │ │ ├── OrientDBClient.java │ │ │ └── package-info.java │ │ └── resources/ │ │ └── log4j.properties │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── OrientDBClientTest.java ├── pom.xml ├── rados/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── RadosClient.java │ │ └── package-info.java │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── RadosClientTest.java ├── redis/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── RedisClient.java │ └── package-info.java ├── rest/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── webservice/ │ │ └── rest/ │ │ ├── RestClient.java │ │ └── package-info.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── webservice/ │ │ └── rest/ │ │ ├── IntegrationTest.java │ │ ├── ResourceLoader.java │ │ ├── RestClientTest.java │ │ ├── RestTestResource.java │ │ └── Utils.java │ └── resources/ │ ├── WebContent/ │ │ └── index.html │ ├── error_trace.txt │ ├── trace.txt │ └── workload_rest ├── riak/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ └── db/ │ │ │ └── riak/ │ │ │ ├── RiakKVClient.java │ │ │ ├── RiakUtils.java │ │ │ └── package-info.java │ │ └── resources/ │ │ └── riak.properties │ └── test/ │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ └── riak/ │ └── RiakKVClientTest.java ├── rocksdb/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── RocksdbClient.java ├── s3/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── conf/ │ │ └── s3.properties │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── S3Client.java │ └── package-info.java ├── solr/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ └── db/ │ │ │ └── solr/ │ │ │ ├── SolrClient.java │ │ │ └── package-info.java │ │ └── resources/ │ │ └── log4j.properties │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── solr/ │ │ ├── SolrClientBaseTest.java │ │ ├── SolrClientCloudTest.java │ │ └── SolrClientTest.java │ └── resources/ │ ├── log4j.properties │ └── solr_config/ │ ├── schema.xml │ └── solrconfig.xml ├── solr6/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── yahoo/ │ │ │ └── ycsb/ │ │ │ └── db/ │ │ │ └── solr6/ │ │ │ ├── SolrClient.java │ │ │ └── package-info.java │ │ └── resources/ │ │ └── log4j.properties │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ └── solr6/ │ │ ├── SolrClientBaseTest.java │ │ ├── SolrClientCloudTest.java │ │ └── SolrClientTest.java │ └── resources/ │ ├── log4j.properties │ └── solr_config/ │ ├── schema.xml │ └── solrconfig.xml ├── tarantool/ │ ├── README.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── conf/ │ │ ├── tarantool-hash.lua │ │ └── tarantool-tree.lua │ └── java/ │ └── com/ │ └── yahoo/ │ └── ycsb/ │ └── db/ │ ├── TarantoolClient.java │ └── package-info.java ├── voldemort/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── conf/ │ │ ├── cluster.xml │ │ ├── server.properties │ │ └── stores.xml │ ├── java/ │ │ └── com/ │ │ └── yahoo/ │ │ └── ycsb/ │ │ └── db/ │ │ ├── VoldemortClient.java │ │ └── package-info.java │ └── resources/ │ └── config/ │ ├── cluster.xml │ ├── server.properties │ └── stores.xml └── workloads/ ├── tsworkload_template ├── tsworkloada ├── workload_template ├── workloada ├── workloadb ├── workloadc ├── workloadd ├── workloade └── workloadf ================================================ FILE CONTENTS ================================================ ================================================ FILE: .editorconfig ================================================ # For more info, see: http://EditorConfig.org root = true [*.java] indent_style = space indent_size = 2 continuation_indent_size = 4 [*.md] indent_style = space indent_size = 2 continuation_indent_size = 4 [*.xml] indent_style = space indent_size = 2 continuation_indent_size = 4 ================================================ FILE: .gitignore ================================================ # ignore compiled byte code target # ignore output files from testing output* # ignore standard Eclipse files .project .classpath .settings .checkstyle # ignore standard IntelliJ files .idea/ *.iml *.ipr *.iws # ignore standard Vim and Emacs temp files *.swp *~ # ignore standard Mac OS X files/dirs .DS_Store ================================================ FILE: .travis.yml ================================================ # Copyright (c) 2010 Yahoo! Inc., 2012 - 2015 YCSB contributors. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # more info here about TravisCI and Java projects # http://docs.travis-ci.com/user/languages/java/ language: java jdk: - oraclejdk9 - oraclejdk8 - openjdk7 addons: hosts: - myshorthost hostname: myshorthost install: mvn install -q -DskipTests=true script: mvn test -q # Services to start for tests. services: - mongodb # temporarily disable riak. failing, docs offline. # - riak # Can't use container based infra because of hosts/hostname sudo: true ================================================ FILE: CONTRIBUTING.md ================================================ ## How To Contribute As more and more databases are created to handle distributed or "cloud" workloads, YCSB needs contributors to write clients to test them. And of course we always need bug fixes, updates for existing databases and new features to keep YCSB going. Here are some guidelines to follow when digging into the code. ## Project Source YCSB is located in a Git repository hosted on GitHub at [https://github.com/brianfrankcooper/YCSB](https://github.com/brianfrankcooper/YCSB). To modify the code, fork the main repo into your own GitHub account or organization and commit changes there. YCSB is written in Java (as most of the new cloud data stores at beginning of the project were written in Java) and is laid out as a multi-module Maven project. You should be able to import the project into your favorite IDE or environment easily. For more details about the Maven layout see the [Guide to Working with Multiple Modules](https://maven.apache.org/guides/mini/guide-multiple-modules.html). ## Licensing YCSB is licensed under the Apache License, Version 2.0 (APL2). Every file included in the project must include the APL header. For example, each Java source file must have a header similar to the following: ```java /** * Copyright (c) 2015-2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ ``` When modifying files that already have a license header, please update the year when you made your edits. E.g. change ``Copyright (c) 2010 Yahoo! Inc., 2012 - 2016 YCSB contributors.`` to ``Copyright (c) 2010 Yahoo! Inc., 2012 - 2017 YCSB contributors.`` If the file only has ``Copyright (c) 2010 Yahoo! Inc.``, append the current year as in ``Copyright (c) 2010 Yahoo! Inc., 2017 YCSB contributors.``. **WARNING**: It should go without saying, but don't copy and paste code from outside authors or sources. If you are a database author and want to copy some example code, it must be APL2 compatible. Client bindings to non-APL databases are perfectly acceptable, as data stores are meant to be used from all kinds of projects. Just make sure not to copy any code or commit libraries or binaries into the YCSB code base. Link to them in the Maven pom file. ## Issues and Support To track bugs, feature requests and releases we use GitHub's integrated [Issues](https://github.com/brianfrankcooper/YCSB/issues). If you find a bug or problem, open an issue with a descriptive title and as many details as you can give us in the body (stack traces, log files, etc). Then if you can create a fix, follow the PR guidelines below. **Note** Before embarking on a code change or DB, search through the existing issues and pull requests to see if anyone is already working on it. Reach out to them if so. For general support, please use the mailing list hosted (of course) with Yahoo groups at [http://groups.yahoo.com/group/ycsb-users](http://groups.yahoo.com/group/ycsb-users). ## Code Style A Java coding style guide is enforced via the Maven CheckStyle plugin. We try not to be too draconian with enforcement but the biggies include: * Whitespaces instead of tabs. * Proper Javadocs for methods and classes. * Camel case member names. * Upper camel case classes and method names. * Line length. CheckStyle will run for pull requests or if you create a package locally so if you just compile and push a commit, you may be surprised when the build fails with a style issue. Just execute ``mvn checkstyle:checkstyle `` before you open a PR and you should avoid any suprises. ## Platforms Since most data bases aim to support multiple platforms, YCSB aims to run on as many as possible as well. Besides **Linux** and **macOS**, YCSB must compile and run for **Windows**. While not all DBs will run under every platform, the YCSB tool itself must be able to execute on all of these systems and hopefully be able to communicate with remote data stores. Additionally, YCSB is targeting Java 7 (1.7.0) as its build version as some users are glacially slow moving to Java 8. So please avoid those Lambdas and Streams for now. ## Pull Requests You've written some amazing code and are excited to share it with the community! It's time to open a PR! Here's what you should do. * Checkout YCSB's ``master`` branch in your own fork and create a new branch based off of it with a name that is reflective of your work. E.g. ``i123`` for fixing an issue or ``db_xyz`` when working on a binding. * Add your changes to the branch. * Commit the code and start the commit message with the component you are working on in square braces. E.g. ``[core] Add another format for exporting histograms.`` or ``[hbase12] Fix interrupted exception bug.``. * Push to your fork and click the ``Create Pull Request`` button. * Wait for the build to complete in the CI pipeline. If it fails with a red X, click through the logs for details and fix any issues and commit your changes. * If you have made changes, please flatten the commits so that the commit logs are nice and clean. Just run a ``git rebase -i ``. After you have opened your PR, a YCSB maintainer will review it and offer constructive feedback via the GitHub review feature. If no one has responded to your PR, please bump the thread by adding comments. **NOTE**: For maintainers, please get another maintainer to sign off on your changes before merging a PR. And if you're writing code, please do create a PR from your fork, don't just push code directly to the master branch. ## Core, Bindings and Workloads The main components of the code base include the core library and benchmarking utility, various database client bindings and workload classes and definitions. ### Core When working on the core classes, keep in mind the following: * Do not change the core behavior or operation of the main benchmarking classes (Particularly the Client and Workload classes). YCSB is used all over the place because it's a consistent standard that allows different users to compare results with the same workloads. If you find a way to drastically improve throughput, that's great! But please check with the rest of the maintainers to see if we can add the tweaks without invalidating years of benchmarks. * Do not remove or modify measurements. Users may have tooling to parse the outputs so if you take something out, they'll be a wee bit unhappy. Extending or adding measurements is fine (so if you do have tooling, expect additions.) * Do not modify existing generators. Again we don't want to invalidate years of benchmarks. Instead, create a new generator or option that can be enabled explicitly (not implicitly!) for users to try out. * Utility classes and methods are welcome. But if they're only ever used by a specific database binding, co-locate the code with that binding. * Don't change the DB interface if at all possible. Implementations can squeeze all kinds of workloads through the existing interface and while it may be easy to change the bindings included with the source code, some users may have private clients they can't share with the community. ### Bindings and Clients When a new database is released a *binding* can be created that implements a client communicating with the given data store that will execute YCSB workloads. Details about writing a DB binding can be found on our [GitHub Wiki page](https://github.com/brianfrankcooper/YCSB/wiki/Adding-a-Database). Some development guidelines to follow include: * Create a new Maven module for your binding. Follow the existing bindings as examples. * The module *must* include a README.md file with details such as: * Database setup with links to documentation so that the YCSB benchmarks will execute properly. * Example command line executions (workload selection, etc). * Required and optional properties (e.g. connection strings, behavior settings, etc) along with the default values. * Versions of the database the binding supports. * Javadoc the binding and all of the methods. Tell us what it does and how it works. Because YCSB is a utility to compare multiple data stores, we need each binding to behave similarly by default. That means each data store should enforce the strictest consistency guarantees available and avoid client side buffering or optimizations. This allows users to evaluate different DBs with a common baseline and tough standards. However you *should* include parameters to tune and improve performance as much as possible to reach those flashy marketing numbers. Just be honest and document what the settings do and what trade-offs are made. (e.g. client side buffering reduces I/O but a crash can lead to data loss). ### Workloads YCSB began comparing various key/value data stores with simple CRUD operations. However as DBs have become more specialized we've added more workloads for various tasks and would love to have more in the future. Keep the following in mind: * Make sure more than one publicly available database can handle your workload. It's no fun if only one player is in the game. * Use the existing DB interface to pass your data around. If you really need another API, discuss with the maintainers to see if there isn't a workaround. * Provide real-world use cases for the workload, not just theoretical idealizations. ================================================ FILE: LICENSE.txt ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: NOTICE.txt ================================================ ========================================================================= NOTICE file for use with, and corresponding to Section 4 of, the Apache License, Version 2.0, in this case for the YCSB project. ========================================================================= This product includes software developed by Yahoo! Inc. (www.yahoo.com) Copyright (c) 2010 Yahoo! Inc. All rights reserved. This product includes software developed by Google Inc. (www.google.com) Copyright (c) 2015 Google Inc. All rights reserved. ================================================ FILE: README.md ================================================ # ByteIterator 从数据库取数据，我使用了ByteIterator数据接口，其例子和好处如下。 代码示例： ```Java @Override public Status read(String table, String key, Set fields, Map result) { try { byte[] value = db.get(key.getBytes()); Map deserialized = deserialize(value); result.putAll(deserialized); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); return Status.ERROR; } return Status.OK; } ``` Why I use ByteIterator here? a.出于性能考虑，主要考虑字符串的成本、拷贝转码问题，流可能是一个图片（blob形式） b.byte是字节，可以屏蔽utf8、gbk等编码细节。文本从磁盘拿出来本来是二进制，需要通过编码转化为对应的字符。ByteIterator可以屏蔽不同服务器编码不一样的的问题。 使用Byte来存储数据有什么缺点和优点？ 略 使用Iterator有什么缺点和优点？可以屏蔽细节？ 略 使用ByteIerator来有什么缺点和有点？ 略 Leveldb and Rocksdb modules of YCSB ==================================== [![Build Status](https://travis-ci.org/brianfrankcooper/YCSB.png?branch=master)](https://travis-ci.org/brianfrankcooper/YCSB) Links ----- http://wiki.github.com/brianfrankcooper/YCSB/ https://labs.yahoo.com/news/yahoo-cloud-serving-benchmark/ ycsb-users@yahoogroups.com Getting Started --------------- 1. Download the [latest release of YCSB](https://github.com/brianfrankcooper/YCSB/releases/latest): ```sh curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.12.0/ycsb-0.12.0.tar.gz tar xfvz ycsb-0.12.0.tar.gz cd ycsb-0.12.0 ``` 2. Set up a database to benchmark. There is a README file under each binding directory. 3. Run YCSB command. On Linux: ```sh bin/ycsb.sh load basic -P workloads/workloada bin/ycsb.sh run basic -P workloads/workloada ``` On Windows: ```bat bin/ycsb.bat load basic -P workloads\workloada bin/ycsb.bat run basic -P workloads\workloada ``` Running the `ycsb` command without any argument will print the usage. See https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload for a detailed documentation on how to run a workload. See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for the list of available workload properties. Building from source -------------------- YCSB requires the use of Maven 3; if you use Maven 2, you may see [errors such as these](https://github.com/brianfrankcooper/YCSB/issues/406). To build the full distribution, with all database bindings: mvn clean package To build a single database binding: mvn -pl com.yahoo.ycsb:mongodb-binding -am clean package ================================================ FILE: Todo.md ================================================ @Override public Status read(String table, String key, Set fields, Map result) { try { byte[] value = db.get(key.getBytes()); Map deserialized = deserialize(value); result.putAll(deserialized); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); return Status.ERROR; } return Status.OK; } /** 1. Why I use ByteIterator here? a.出于性能考虑，主要考虑字符串的成本、拷贝转码问题，流可能是一个图片（blob形式） b.byte是字节可以，可以屏蔽utf8、gbk等编码细节。文本从磁盘拿出来本来是二进制，需要通过编码转化为对应的字符。 ByteIterator可以屏蔽不同服务器编码不一样的的问题。 **/ ================================================ FILE: accumulo1.6/README.md ================================================ ## Quick Start This section describes how to run YCSB on [Accumulo](https://accumulo.apache.org/). ### 1. Start Accumulo See the [Accumulo Documentation](https://accumulo.apache.org/1.6/accumulo_user_manual.html#_installation) for details on installing and running Accumulo. Before running the YCSB test you must create the Accumulo table. Again see the [Accumulo Documentation](https://accumulo.apache.org/1.6/accumulo_user_manual.html#_basic_administration) for details. The default table name is `ycsb`. ### 2. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:accumulo1.6-binding -am clean package ### 3. Create the Accumulo table By default, YCSB uses a table with the name "usertable". Users must create this table before loading data into Accumulo. For maximum Accumulo performance, the Accumulo table must be pre-split. A simple Ruby script, based on the HBase README, can generate adequate split-point. 10's of Tablets per TabletServer is a good starting point. Unless otherwise specified, the following commands should run on any version of Accumulo. $ echo 'num_splits = 20; puts (1..num_splits).map {|i| "user#{1000+i*(9999-1000)/num_splits}"}' | ruby > /tmp/splits.txt $ accumulo shell -u -p -e "createtable usertable" $ accumulo shell -u -p -e "addsplits -t usertable -sf /tmp/splits.txt" $ accumulo shell -u -p -e "config -t usertable -s table.cache.block.enable=true" Additionally, there are some other configuration properties which can increase performance. These can be set on the Accumulo table via the shell after it is created. Setting the table durability to `flush` relaxes the constraints on data durability during hard power-outages (avoids calls to fsync). Accumulo defaults table compression to `gzip` which is not particularly fast; `snappy` is a faster and similarly-efficient option. The mutation queue property controls how many writes that Accumulo will buffer in memory before performing a flush; this property should be set relative to the amount of JVM heap the TabletServers are given. Please note that the `table.durability` and `tserver.total.mutation.queue.max` properties only exists for >=Accumulo-1.7. There are no concise replacements for these properties in earlier versions. accumulo> config -s table.durability=flush accumulo> config -s tserver.total.mutation.queue.max=256M accumulo> config -t usertable -s table.file.compress.type=snappy On repeated data loads, the following commands may be helpful to re-set the state of the table quickly. accumulo> createtable tmp --copy-splits usertable --copy-config usertable accumulo> deletetable --force usertable accumulo> renametable tmp usertable accumulo> compact --wait -t accumulo.metadata ### 4. Load Data and Run Tests Load the data: ./bin/ycsb load accumulo1.6 -s -P workloads/workloada \ -p accumulo.zooKeepers=localhost \ -p accumulo.columnFamily=ycsb \ -p accumulo.instanceName=ycsb \ -p accumulo.username=user \ -p accumulo.password=supersecret \ > outputLoad.txt Run the workload test: ./bin/ycsb run accumulo1.6 -s -P workloads/workloada \ -p accumulo.zooKeepers=localhost \ -p accumulo.columnFamily=ycsb \ -p accumulo.instanceName=ycsb \ -p accumulo.username=user \ -p accumulo.password=supersecret \ > outputLoad.txt ## Accumulo Configuration Parameters - `accumulo.zooKeepers` - The Accumulo cluster's [zookeeper servers](https://accumulo.apache.org/1.6/accumulo_user_manual.html#_connecting). - Should contain a comma separated list of of hostname or hostname:port values. - No default value. - `accumulo.columnFamily` - The name of the column family to use to store the data within the table. - No default value. - `accumulo.instanceName` - Name of the Accumulo [instance](https://accumulo.apache.org/1.6/accumulo_user_manual.html#_connecting). - No default value. - `accumulo.username` - The username to use when connecting to Accumulo. - No default value. - `accumulo.password` - The password for the user connecting to Accumulo. - No default value. ================================================ FILE: accumulo1.6/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent accumulo1.6-binding Accumulo 1.6 DB Binding 2.2.0 true org.apache.accumulo accumulo-core ${accumulo.1.6.version} org.apache.hadoop hadoop-common ${hadoop.version} jdk.tools jdk.tools com.yahoo.ycsb core ${project.version} provided junit junit 4.12 test org.apache.accumulo accumulo-minicluster ${accumulo.1.6.version} test org.slf4j slf4j-api 1.7.13 ../workloads workloads src/test/resources ================================================ FILE: accumulo1.6/src/main/conf/accumulo.properties ================================================ # Copyright 2014 Cloudera, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Sample Accumulo configuration properties # # You may either set properties here or via the command line. # # This will influence the keys we write accumulo.columnFamily=YCSB # This should be set based on your Accumulo cluster #accumulo.instanceName=ExampleInstance # Comma separated list of host:port tuples for the ZooKeeper quorum used # by your Accumulo cluster #accumulo.zooKeepers=zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181 # This user will need permissions on the table YCSB works against #accumulo.username=ycsb #accumulo.password=protectyaneck # Controls how long our client writer will wait to buffer more data # measured in milliseconds accumulo.batchWriterMaxLatency=30000 # Controls how much data our client will attempt to buffer before sending # measured in bytes accumulo.batchWriterSize=100000 # Controls how many worker threads our client will use to parallelize writes accumulo.batchWriterThreads=1 ================================================ FILE: accumulo1.6/src/main/java/com/yahoo/ycsb/db/accumulo/AccumuloClient.java ================================================ /** * Copyright (c) 2011 YCSB++ project, 2014-2016 YCSB contributors. * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.accumulo; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.SortedMap; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CleanUp; import org.apache.hadoop.io.Text; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; /** * Accumulo binding for YCSB. */ public class AccumuloClient extends DB { private ZooKeeperInstance inst; private Connector connector; private Text colFam = new Text(""); private byte[] colFamBytes = new byte[0]; private final ConcurrentHashMap writers = new ConcurrentHashMap<>(); static { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { CleanUp.shutdownNow(); } }); } @Override public void init() throws DBException { colFam = new Text(getProperties().getProperty("accumulo.columnFamily")); colFamBytes = colFam.toString().getBytes(UTF_8); inst = new ZooKeeperInstance( getProperties().getProperty("accumulo.instanceName"), getProperties().getProperty("accumulo.zooKeepers")); try { String principal = getProperties().getProperty("accumulo.username"); AuthenticationToken token = new PasswordToken(getProperties().getProperty("accumulo.password")); connector = inst.getConnector(principal, token); } catch (AccumuloException | AccumuloSecurityException e) { throw new DBException(e); } if (!(getProperties().getProperty("accumulo.pcFlag", "none").equals("none"))) { System.err.println("Sorry, the ZK based producer/consumer implementation has been removed. " + "Please see YCSB issue #416 for work on adding a general solution to coordinated work."); } } @Override public void cleanup() throws DBException { try { Iterator iterator = writers.values().iterator(); while (iterator.hasNext()) { BatchWriter writer = iterator.next(); writer.close(); iterator.remove(); } } catch (MutationsRejectedException e) { throw new DBException(e); } } /** * Called when the user specifies a table that isn't the same as the existing * table. Connect to it and if necessary, close our current connection. * * @param table * The table to open. */ public BatchWriter getWriter(String table) throws TableNotFoundException { // tl;dr We're paying a cost for the ConcurrentHashMap here to deal with the DB api. // We know that YCSB is really only ever going to send us data for one table, so using // a concurrent data structure is overkill (especially in such a hot code path). // However, the impact seems to be relatively negligible in trivial local tests and it's // "more correct" WRT to the API. BatchWriter writer = writers.get(table); if (null == writer) { BatchWriter newWriter = createBatchWriter(table); BatchWriter oldWriter = writers.putIfAbsent(table, newWriter); // Someone beat us to creating a BatchWriter for this table, use their BatchWriters if (null != oldWriter) { try { // Make sure to clean up our new batchwriter! newWriter.close(); } catch (MutationsRejectedException e) { throw new RuntimeException(e); } writer = oldWriter; } else { writer = newWriter; } } return writer; } /** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); } /** * Gets a scanner from Accumulo over one row. * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(String table, Text row, Set fields) throws TableNotFoundException { Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(row)); if (fields != null) { for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } return scanner; } @Override public Status read(String table, String key, Set fields, Map result) { Scanner scanner = null; try { scanner = getRow(table, new Text(key), null); // Pick out the results we care about. final Text cq = new Text(); for (Entry entry : scanner) { entry.getKey().getColumnQualifier(cq); Value v = entry.getValue(); byte[] buf = v.get(); result.put(cq.toString(), new ByteArrayByteIterator(buf)); } } catch (Exception e) { System.err.println("Error trying to reading Accumulo table " + table + " " + key); e.printStackTrace(); return Status.ERROR; } finally { if (null != scanner) { scanner.close(); } } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // Just make the end 'infinity' and only read as much as we need. Scanner scanner = null; try { scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(new Text(startkey), null)); // Have Accumulo send us complete rows, serialized in a single Key-Value pair IteratorSetting cfg = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(cfg); // If no fields are provided, we assume one column/row. if (fields != null) { // And add each of them as fields we want. for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } int count = 0; for (Entry entry : scanner) { // Deserialize the row SortedMap row = WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()); HashMap rowData; if (null != fields) { rowData = new HashMap<>(fields.size()); } else { rowData = new HashMap<>(); } result.add(rowData); // Parse the data in the row, avoid unnecessary Text object creation final Text cq = new Text(); for (Entry rowEntry : row.entrySet()) { rowEntry.getKey().getColumnQualifier(cq); rowData.put(cq.toString(), new ByteArrayByteIterator(rowEntry.getValue().get())); } if (count++ == recordcount) { // Done reading the last row. break; } } } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table."); e.printStackTrace(); return Status.ERROR; } catch (IOException e) { System.err.println("Error deserializing data from Accumulo."); e.printStackTrace(); return Status.ERROR; } finally { if (null != scanner) { scanner.close(); } } return Status.OK; } @Override public Status update(String table, String key, Map values) { BatchWriter bw = null; try { bw = getWriter(table); } catch (TableNotFoundException e) { System.err.println("Error opening batch writer to Accumulo table " + table); e.printStackTrace(); return Status.ERROR; } Mutation mutInsert = new Mutation(key.getBytes(UTF_8)); for (Map.Entry entry : values.entrySet()) { mutInsert.put(colFamBytes, entry.getKey().getBytes(UTF_8), entry.getValue().toArray()); } try { bw.addMutation(mutInsert); } catch (MutationsRejectedException e) { System.err.println("Error performing update."); e.printStackTrace(); return Status.ERROR; } return Status.BATCHED_OK; } @Override public Status insert(String t, String key, Map values) { return update(t, key, values); } @Override public Status delete(String table, String key) { BatchWriter bw; try { bw = getWriter(table); } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table."); e.printStackTrace(); return Status.ERROR; } try { deleteRow(table, new Text(key), bw); } catch (TableNotFoundException | MutationsRejectedException e) { System.err.println("Error performing delete."); e.printStackTrace(); return Status.ERROR; } catch (RuntimeException e) { System.err.println("Error performing delete."); e.printStackTrace(); return Status.ERROR; } return Status.OK; } // These functions are adapted from RowOperations.java: private void deleteRow(String table, Text row, BatchWriter bw) throws MutationsRejectedException, TableNotFoundException { // TODO Use a batchDeleter instead deleteRow(getRow(table, row, null), bw); } /** * Deletes a row, given a Scanner of JUST that row. */ private void deleteRow(Scanner scanner, BatchWriter bw) throws MutationsRejectedException { Mutation deleter = null; // iterate through the keys final Text row = new Text(); final Text cf = new Text(); final Text cq = new Text(); for (Entry entry : scanner) { // create a mutation for the row if (deleter == null) { entry.getKey().getRow(row); deleter = new Mutation(row); } entry.getKey().getColumnFamily(cf); entry.getKey().getColumnQualifier(cq); // the remove function adds the key with the delete flag set to true deleter.putDelete(cf, cq); } bw.addMutation(deleter); } } ================================================ FILE: accumulo1.6/src/main/java/com/yahoo/ycsb/db/accumulo/package-info.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for Apache Accumulo. */ package com.yahoo.ycsb.db.accumulo; ================================================ FILE: accumulo1.6/src/test/java/com/yahoo/ycsb/db/accumulo/AccumuloTest.java ================================================ /* * Copyright (c) 2016 YCSB contributors. * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.accumulo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.util.Map.Entry; import java.util.Properties; import com.yahoo.ycsb.Workload; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.workloads.CoreWorkload; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Use an Accumulo MiniCluster to test out basic workload operations with * the Accumulo binding. */ public class AccumuloTest { private static final Logger LOG = LoggerFactory.getLogger(AccumuloTest.class); private static final int INSERT_COUNT = 2000; private static final int TRANSACTION_COUNT = 2000; @ClassRule public static TemporaryFolder workingDir = new TemporaryFolder(); @Rule public TestName test = new TestName(); private static MiniAccumuloCluster cluster; private static Properties properties; private Workload workload; private DB client; private Properties workloadProps; private static boolean isWindows() { final String os = System.getProperty("os.name"); return os.startsWith("Windows"); } @BeforeClass public static void setup() throws Exception { // Minicluster setup fails on Windows with an UnsatisfiedLinkError. // Skip if windows. assumeTrue(!isWindows()); cluster = new MiniAccumuloCluster(workingDir.newFolder("accumulo").getAbsoluteFile(), "protectyaneck"); LOG.debug("starting minicluster"); cluster.start(); LOG.debug("creating connection for admin operations."); // set up the table and user final Connector admin = cluster.getConnector("root", "protectyaneck"); admin.tableOperations().create(CoreWorkload.TABLENAME_PROPERTY_DEFAULT); admin.securityOperations().createLocalUser("ycsb", new PasswordToken("protectyaneck")); admin.securityOperations().grantTablePermission("ycsb", CoreWorkload.TABLENAME_PROPERTY_DEFAULT, TablePermission.READ); admin.securityOperations().grantTablePermission("ycsb", CoreWorkload.TABLENAME_PROPERTY_DEFAULT, TablePermission.WRITE); // set properties the binding will read properties = new Properties(); properties.setProperty("accumulo.zooKeepers", cluster.getZooKeepers()); properties.setProperty("accumulo.instanceName", cluster.getInstanceName()); properties.setProperty("accumulo.columnFamily", "family"); properties.setProperty("accumulo.username", "ycsb"); properties.setProperty("accumulo.password", "protectyaneck"); // cut down the batch writer timeout so that writes will push through. properties.setProperty("accumulo.batchWriterMaxLatency", "4"); // set these explicitly to the defaults at the time we're compiled, since they'll be inlined in our class. properties.setProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT); properties.setProperty(CoreWorkload.FIELD_COUNT_PROPERTY, CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT); properties.setProperty(CoreWorkload.INSERT_ORDER_PROPERTY, "ordered"); } @AfterClass public static void clusterCleanup() throws Exception { if (cluster != null) { LOG.debug("shutting down minicluster"); cluster.stop(); cluster = null; } } @Before public void client() throws Exception { LOG.debug("Loading workload properties for {}", test.getMethodName()); workloadProps = new Properties(); workloadProps.load(getClass().getResourceAsStream("/workloads/" + test.getMethodName())); for (String prop : properties.stringPropertyNames()) { workloadProps.setProperty(prop, properties.getProperty(prop)); } // TODO we need a better test rig for 'run this ycsb workload' LOG.debug("initializing measurements and workload"); Measurements.setProperties(workloadProps); workload = new CoreWorkload(); workload.init(workloadProps); LOG.debug("initializing client"); client = new AccumuloClient(); client.setProperties(workloadProps); client.init(); } @After public void cleanup() throws Exception { if (client != null) { LOG.debug("cleaning up client"); client.cleanup(); client = null; } if (workload != null) { LOG.debug("cleaning up workload"); workload.cleanup(); } } @After public void truncateTable() throws Exception { if (cluster != null) { LOG.debug("truncating table {}", CoreWorkload.TABLENAME_PROPERTY_DEFAULT); final Connector admin = cluster.getConnector("root", "protectyaneck"); admin.tableOperations().deleteRows(CoreWorkload.TABLENAME_PROPERTY_DEFAULT, null, null); } } @Test public void workloada() throws Exception { runWorkload(); } @Test public void workloadb() throws Exception { runWorkload(); } @Test public void workloadc() throws Exception { runWorkload(); } @Test public void workloadd() throws Exception { runWorkload(); } @Test public void workloade() throws Exception { runWorkload(); } /** * go through a workload cycle. *

    *
  1. initialize thread-specific state *
  2. load the workload dataset *
  3. run workload transactions *
*/ private void runWorkload() throws Exception { final Object state = workload.initThread(workloadProps,0,0); LOG.debug("load"); for (int i = 0; i < INSERT_COUNT; i++) { assertTrue("insert failed.", workload.doInsert(client, state)); } // Ensure we wait long enough for the batch writer to flush // TODO accumulo client should be flushing per insert by default. Thread.sleep(2000); LOG.debug("verify number of cells"); final Scanner scanner = cluster.getConnector("root", "protectyaneck").createScanner(CoreWorkload.TABLENAME_PROPERTY_DEFAULT, Authorizations.EMPTY); int count = 0; for (Entry entry : scanner) { count++; } assertEquals("Didn't get enough total cells.", (Integer.valueOf(CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT) * INSERT_COUNT), count); LOG.debug("run"); for (int i = 0; i < TRANSACTION_COUNT; i++) { assertTrue("transaction failed.", workload.doTransaction(client, state)); } } } ================================================ FILE: accumulo1.6/src/test/resources/log4j.properties ================================================ # # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=INFO, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZooKeeper log4j.logger.com.yahoo.ycsb.db.accumulo=INFO log4j.logger.org.apache.zookeeper=ERROR log4j.logger.org.apache.accumulo=WARN ================================================ FILE: accumulo1.7/README.md ================================================ ## Quick Start This section describes how to run YCSB on [Accumulo](https://accumulo.apache.org/). ### 1. Start Accumulo See the [Accumulo Documentation](https://accumulo.apache.org/1.7/accumulo_user_manual.html#_installation) for details on installing and running Accumulo. Before running the YCSB test you must create the Accumulo table. Again see the [Accumulo Documentation](https://accumulo.apache.org/1.7/accumulo_user_manual.html#_basic_administration) for details. The default table name is `ycsb`. ### 2. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:accumulo1.7-binding -am clean package ### 3. Create the Accumulo table By default, YCSB uses a table with the name "usertable". Users must create this table before loading data into Accumulo. For maximum Accumulo performance, the Accumulo table must be pre-split. A simple Ruby script, based on the HBase README, can generate adequate split-point. 10's of Tablets per TabletServer is a good starting point. Unless otherwise specified, the following commands should run on any version of Accumulo. $ echo 'num_splits = 20; puts (1..num_splits).map {|i| "user#{1000+i*(9999-1000)/num_splits}"}' | ruby > /tmp/splits.txt $ accumulo shell -u -p -e "createtable usertable" $ accumulo shell -u -p -e "addsplits -t usertable -sf /tmp/splits.txt" $ accumulo shell -u -p -e "config -t usertable -s table.cache.block.enable=true" Additionally, there are some other configuration properties which can increase performance. These can be set on the Accumulo table via the shell after it is created. Setting the table durability to `flush` relaxes the constraints on data durability during hard power-outages (avoids calls to fsync). Accumulo defaults table compression to `gzip` which is not particularly fast; `snappy` is a faster and similarly-efficient option. The mutation queue property controls how many writes that Accumulo will buffer in memory before performing a flush; this property should be set relative to the amount of JVM heap the TabletServers are given. Please note that the `table.durability` and `tserver.total.mutation.queue.max` properties only exists for >=Accumulo-1.7. There are no concise replacements for these properties in earlier versions. accumulo> config -s table.durability=flush accumulo> config -s tserver.total.mutation.queue.max=256M accumulo> config -t usertable -s table.file.compress.type=snappy On repeated data loads, the following commands may be helpful to re-set the state of the table quickly. accumulo> createtable tmp --copy-splits usertable --copy-config usertable accumulo> deletetable --force usertable accumulo> renametable tmp usertable accumulo> compact --wait -t accumulo.metadata ### 4. Load Data and Run Tests Load the data: ./bin/ycsb load accumulo1.7 -s -P workloads/workloada \ -p accumulo.zooKeepers=localhost \ -p accumulo.columnFamily=ycsb \ -p accumulo.instanceName=ycsb \ -p accumulo.username=user \ -p accumulo.password=supersecret \ > outputLoad.txt Run the workload test: ./bin/ycsb run accumulo1.7 -s -P workloads/workloada \ -p accumulo.zooKeepers=localhost \ -p accumulo.columnFamily=ycsb \ -p accumulo.instanceName=ycsb \ -p accumulo.username=user \ -p accumulo.password=supersecret \ > outputLoad.txt ## Accumulo Configuration Parameters - `accumulo.zooKeepers` - The Accumulo cluster's [zookeeper servers](https://accumulo.apache.org/1.7/accumulo_user_manual.html#_connecting). - Should contain a comma separated list of of hostname or hostname:port values. - No default value. - `accumulo.columnFamily` - The name of the column family to use to store the data within the table. - No default value. - `accumulo.instanceName` - Name of the Accumulo [instance](https://accumulo.apache.org/1.7/accumulo_user_manual.html#_connecting). - No default value. - `accumulo.username` - The username to use when connecting to Accumulo. - No default value. - `accumulo.password` - The password for the user connecting to Accumulo. - No default value. ================================================ FILE: accumulo1.7/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent accumulo1.7-binding Accumulo 1.7 DB Binding 2.2.0 true org.apache.accumulo accumulo-core ${accumulo.1.7.version} org.apache.hadoop hadoop-common ${hadoop.version} jdk.tools jdk.tools com.yahoo.ycsb core ${project.version} provided junit junit 4.12 test org.apache.accumulo accumulo-minicluster ${accumulo.1.7.version} test org.slf4j slf4j-api 1.7.13 ../workloads workloads src/test/resources ================================================ FILE: accumulo1.7/src/main/conf/accumulo.properties ================================================ # Copyright 2014 Cloudera, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Sample Accumulo configuration properties # # You may either set properties here or via the command line. # # This will influence the keys we write accumulo.columnFamily=YCSB # This should be set based on your Accumulo cluster #accumulo.instanceName=ExampleInstance # Comma separated list of host:port tuples for the ZooKeeper quorum used # by your Accumulo cluster #accumulo.zooKeepers=zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181 # This user will need permissions on the table YCSB works against #accumulo.username=ycsb #accumulo.password=protectyaneck # Controls how long our client writer will wait to buffer more data # measured in milliseconds accumulo.batchWriterMaxLatency=30000 # Controls how much data our client will attempt to buffer before sending # measured in bytes accumulo.batchWriterSize=100000 # Controls how many worker threads our client will use to parallelize writes accumulo.batchWriterThreads=1 ================================================ FILE: accumulo1.7/src/main/java/com/yahoo/ycsb/db/accumulo/AccumuloClient.java ================================================ /** * Copyright (c) 2011 YCSB++ project, 2014-2016 YCSB contributors. * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.accumulo; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.SortedMap; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CleanUp; import org.apache.hadoop.io.Text; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; /** * Accumulo binding for YCSB. */ public class AccumuloClient extends DB { private ZooKeeperInstance inst; private Connector connector; private Text colFam = new Text(""); private byte[] colFamBytes = new byte[0]; private final ConcurrentHashMap writers = new ConcurrentHashMap<>(); static { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { CleanUp.shutdownNow(); } }); } @Override public void init() throws DBException { colFam = new Text(getProperties().getProperty("accumulo.columnFamily")); colFamBytes = colFam.toString().getBytes(UTF_8); inst = new ZooKeeperInstance(new ClientConfiguration() .withInstance(getProperties().getProperty("accumulo.instanceName")) .withZkHosts(getProperties().getProperty("accumulo.zooKeepers"))); try { String principal = getProperties().getProperty("accumulo.username"); AuthenticationToken token = new PasswordToken(getProperties().getProperty("accumulo.password")); connector = inst.getConnector(principal, token); } catch (AccumuloException | AccumuloSecurityException e) { throw new DBException(e); } if (!(getProperties().getProperty("accumulo.pcFlag", "none").equals("none"))) { System.err.println("Sorry, the ZK based producer/consumer implementation has been removed. " + "Please see YCSB issue #416 for work on adding a general solution to coordinated work."); } } @Override public void cleanup() throws DBException { try { Iterator iterator = writers.values().iterator(); while (iterator.hasNext()) { BatchWriter writer = iterator.next(); writer.close(); iterator.remove(); } } catch (MutationsRejectedException e) { throw new DBException(e); } } /** * Called when the user specifies a table that isn't the same as the existing * table. Connect to it and if necessary, close our current connection. * * @param table * The table to open. */ public BatchWriter getWriter(String table) throws TableNotFoundException { // tl;dr We're paying a cost for the ConcurrentHashMap here to deal with the DB api. // We know that YCSB is really only ever going to send us data for one table, so using // a concurrent data structure is overkill (especially in such a hot code path). // However, the impact seems to be relatively negligible in trivial local tests and it's // "more correct" WRT to the API. BatchWriter writer = writers.get(table); if (null == writer) { BatchWriter newWriter = createBatchWriter(table); BatchWriter oldWriter = writers.putIfAbsent(table, newWriter); // Someone beat us to creating a BatchWriter for this table, use their BatchWriters if (null != oldWriter) { try { // Make sure to clean up our new batchwriter! newWriter.close(); } catch (MutationsRejectedException e) { throw new RuntimeException(e); } writer = oldWriter; } else { writer = newWriter; } } return writer; } /** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); } /** * Gets a scanner from Accumulo over one row. * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(String table, Text row, Set fields) throws TableNotFoundException { Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(row)); if (fields != null) { for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } return scanner; } @Override public Status read(String table, String key, Set fields, Map result) { Scanner scanner = null; try { scanner = getRow(table, new Text(key), null); // Pick out the results we care about. final Text cq = new Text(); for (Entry entry : scanner) { entry.getKey().getColumnQualifier(cq); Value v = entry.getValue(); byte[] buf = v.get(); result.put(cq.toString(), new ByteArrayByteIterator(buf)); } } catch (Exception e) { System.err.println("Error trying to reading Accumulo table " + table + " " + key); e.printStackTrace(); return Status.ERROR; } finally { if (null != scanner) { scanner.close(); } } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // Just make the end 'infinity' and only read as much as we need. Scanner scanner = null; try { scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(new Text(startkey), null)); // Have Accumulo send us complete rows, serialized in a single Key-Value pair IteratorSetting cfg = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(cfg); // If no fields are provided, we assume one column/row. if (fields != null) { // And add each of them as fields we want. for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } int count = 0; for (Entry entry : scanner) { // Deserialize the row SortedMap row = WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()); HashMap rowData; if (null != fields) { rowData = new HashMap<>(fields.size()); } else { rowData = new HashMap<>(); } result.add(rowData); // Parse the data in the row, avoid unnecessary Text object creation final Text cq = new Text(); for (Entry rowEntry : row.entrySet()) { rowEntry.getKey().getColumnQualifier(cq); rowData.put(cq.toString(), new ByteArrayByteIterator(rowEntry.getValue().get())); } if (count++ == recordcount) { // Done reading the last row. break; } } } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table."); e.printStackTrace(); return Status.ERROR; } catch (IOException e) { System.err.println("Error deserializing data from Accumulo."); e.printStackTrace(); return Status.ERROR; } finally { if (null != scanner) { scanner.close(); } } return Status.OK; } @Override public Status update(String table, String key, Map values) { BatchWriter bw = null; try { bw = getWriter(table); } catch (TableNotFoundException e) { System.err.println("Error opening batch writer to Accumulo table " + table); e.printStackTrace(); return Status.ERROR; } Mutation mutInsert = new Mutation(key.getBytes(UTF_8)); for (Map.Entry entry : values.entrySet()) { mutInsert.put(colFamBytes, entry.getKey().getBytes(UTF_8), entry.getValue().toArray()); } try { bw.addMutation(mutInsert); } catch (MutationsRejectedException e) { System.err.println("Error performing update."); e.printStackTrace(); return Status.ERROR; } return Status.BATCHED_OK; } @Override public Status insert(String t, String key, Map values) { return update(t, key, values); } @Override public Status delete(String table, String key) { BatchWriter bw; try { bw = getWriter(table); } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table."); e.printStackTrace(); return Status.ERROR; } try { deleteRow(table, new Text(key), bw); } catch (TableNotFoundException | MutationsRejectedException e) { System.err.println("Error performing delete."); e.printStackTrace(); return Status.ERROR; } catch (RuntimeException e) { System.err.println("Error performing delete."); e.printStackTrace(); return Status.ERROR; } return Status.OK; } // These functions are adapted from RowOperations.java: private void deleteRow(String table, Text row, BatchWriter bw) throws MutationsRejectedException, TableNotFoundException { // TODO Use a batchDeleter instead deleteRow(getRow(table, row, null), bw); } /** * Deletes a row, given a Scanner of JUST that row. */ private void deleteRow(Scanner scanner, BatchWriter bw) throws MutationsRejectedException { Mutation deleter = null; // iterate through the keys final Text row = new Text(); final Text cf = new Text(); final Text cq = new Text(); for (Entry entry : scanner) { // create a mutation for the row if (deleter == null) { entry.getKey().getRow(row); deleter = new Mutation(row); } entry.getKey().getColumnFamily(cf); entry.getKey().getColumnQualifier(cq); // the remove function adds the key with the delete flag set to true deleter.putDelete(cf, cq); } bw.addMutation(deleter); } } ================================================ FILE: accumulo1.7/src/main/java/com/yahoo/ycsb/db/accumulo/package-info.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for Apache Accumulo. */ package com.yahoo.ycsb.db.accumulo; ================================================ FILE: accumulo1.7/src/test/java/com/yahoo/ycsb/db/accumulo/AccumuloTest.java ================================================ /* * Copyright (c) 2016 YCSB contributors. * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.accumulo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.util.Map.Entry; import java.util.Properties; import com.yahoo.ycsb.Workload; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.workloads.CoreWorkload; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Use an Accumulo MiniCluster to test out basic workload operations with * the Accumulo binding. */ public class AccumuloTest { private static final Logger LOG = LoggerFactory.getLogger(AccumuloTest.class); private static final int INSERT_COUNT = 2000; private static final int TRANSACTION_COUNT = 2000; @ClassRule public static TemporaryFolder workingDir = new TemporaryFolder(); @Rule public TestName test = new TestName(); private static MiniAccumuloCluster cluster; private static Properties properties; private Workload workload; private DB client; private Properties workloadProps; private static boolean isWindows() { final String os = System.getProperty("os.name"); return os.startsWith("Windows"); } @BeforeClass public static void setup() throws Exception { // Minicluster setup fails on Windows with an UnsatisfiedLinkError. // Skip if windows. assumeTrue(!isWindows()); cluster = new MiniAccumuloCluster(workingDir.newFolder("accumulo").getAbsoluteFile(), "protectyaneck"); LOG.debug("starting minicluster"); cluster.start(); LOG.debug("creating connection for admin operations."); // set up the table and user final Connector admin = cluster.getConnector("root", "protectyaneck"); admin.tableOperations().create(CoreWorkload.TABLENAME_PROPERTY_DEFAULT); admin.securityOperations().createLocalUser("ycsb", new PasswordToken("protectyaneck")); admin.securityOperations().grantTablePermission("ycsb", CoreWorkload.TABLENAME_PROPERTY_DEFAULT, TablePermission.READ); admin.securityOperations().grantTablePermission("ycsb", CoreWorkload.TABLENAME_PROPERTY_DEFAULT, TablePermission.WRITE); // set properties the binding will read properties = new Properties(); properties.setProperty("accumulo.zooKeepers", cluster.getZooKeepers()); properties.setProperty("accumulo.instanceName", cluster.getInstanceName()); properties.setProperty("accumulo.columnFamily", "family"); properties.setProperty("accumulo.username", "ycsb"); properties.setProperty("accumulo.password", "protectyaneck"); // cut down the batch writer timeout so that writes will push through. properties.setProperty("accumulo.batchWriterMaxLatency", "4"); // set these explicitly to the defaults at the time we're compiled, since they'll be inlined in our class. properties.setProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT); properties.setProperty(CoreWorkload.FIELD_COUNT_PROPERTY, CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT); properties.setProperty(CoreWorkload.INSERT_ORDER_PROPERTY, "ordered"); } @AfterClass public static void clusterCleanup() throws Exception { if (cluster != null) { LOG.debug("shutting down minicluster"); cluster.stop(); cluster = null; } } @Before public void client() throws Exception { LOG.debug("Loading workload properties for {}", test.getMethodName()); workloadProps = new Properties(); workloadProps.load(getClass().getResourceAsStream("/workloads/" + test.getMethodName())); for (String prop : properties.stringPropertyNames()) { workloadProps.setProperty(prop, properties.getProperty(prop)); } // TODO we need a better test rig for 'run this ycsb workload' LOG.debug("initializing measurements and workload"); Measurements.setProperties(workloadProps); workload = new CoreWorkload(); workload.init(workloadProps); LOG.debug("initializing client"); client = new AccumuloClient(); client.setProperties(workloadProps); client.init(); } @After public void cleanup() throws Exception { if (client != null) { LOG.debug("cleaning up client"); client.cleanup(); client = null; } if (workload != null) { LOG.debug("cleaning up workload"); workload.cleanup(); } } @After public void truncateTable() throws Exception { if (cluster != null) { LOG.debug("truncating table {}", CoreWorkload.TABLENAME_PROPERTY_DEFAULT); final Connector admin = cluster.getConnector("root", "protectyaneck"); admin.tableOperations().deleteRows(CoreWorkload.TABLENAME_PROPERTY_DEFAULT, null, null); } } @Test public void workloada() throws Exception { runWorkload(); } @Test public void workloadb() throws Exception { runWorkload(); } @Test public void workloadc() throws Exception { runWorkload(); } @Test public void workloadd() throws Exception { runWorkload(); } @Test public void workloade() throws Exception { runWorkload(); } /** * go through a workload cycle. *
    *
  1. initialize thread-specific state *
  2. load the workload dataset *
  3. run workload transactions *
*/ private void runWorkload() throws Exception { final Object state = workload.initThread(workloadProps,0,0); LOG.debug("load"); for (int i = 0; i < INSERT_COUNT; i++) { assertTrue("insert failed.", workload.doInsert(client, state)); } // Ensure we wait long enough for the batch writer to flush // TODO accumulo client should be flushing per insert by default. Thread.sleep(2000); LOG.debug("verify number of cells"); final Scanner scanner = cluster.getConnector("root", "protectyaneck").createScanner(CoreWorkload.TABLENAME_PROPERTY_DEFAULT, Authorizations.EMPTY); int count = 0; for (Entry entry : scanner) { count++; } assertEquals("Didn't get enough total cells.", (Integer.valueOf(CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT) * INSERT_COUNT), count); LOG.debug("run"); for (int i = 0; i < TRANSACTION_COUNT; i++) { assertTrue("transaction failed.", workload.doTransaction(client, state)); } } } ================================================ FILE: accumulo1.7/src/test/resources/log4j.properties ================================================ # # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=INFO, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZooKeeper log4j.logger.com.yahoo.ycsb.db.accumulo=DEBUG log4j.logger.org.apache.zookeeper=ERROR log4j.logger.org.apache.accumulo=WARN ================================================ FILE: accumulo1.8/README.md ================================================ ## Quick Start This section describes how to run YCSB on [Accumulo](https://accumulo.apache.org/). ### 1. Start Accumulo See the [Accumulo Documentation](https://accumulo.apache.org/1.8/accumulo_user_manual.html#_installation) for details on installing and running Accumulo. Before running the YCSB test you must create the Accumulo table. Again see the [Accumulo Documentation](https://accumulo.apache.org/1.8/accumulo_user_manual.html#_basic_administration) for details. The default table name is `ycsb`. ### 2. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:accumulo1.8-binding -am clean package ### 3. Create the Accumulo table By default, YCSB uses a table with the name "usertable". Users must create this table before loading data into Accumulo. For maximum Accumulo performance, the Accumulo table must be pre-split. A simple Ruby script, based on the HBase README, can generate adequate split-point. 10's of Tablets per TabletServer is a good starting point. Unless otherwise specified, the following commands should run on any version of Accumulo. $ echo 'num_splits = 20; puts (1..num_splits).map {|i| "user#{1000+i*(9999-1000)/num_splits}"}' | ruby > /tmp/splits.txt $ accumulo shell -u -p -e "createtable usertable" $ accumulo shell -u -p -e "addsplits -t usertable -sf /tmp/splits.txt" $ accumulo shell -u -p -e "config -t usertable -s table.cache.block.enable=true" Additionally, there are some other configuration properties which can increase performance. These can be set on the Accumulo table via the shell after it is created. Setting the table durability to `flush` relaxes the constraints on data durability during hard power-outages (avoids calls to fsync). Accumulo defaults table compression to `gzip` which is not particularly fast; `snappy` is a faster and similarly-efficient option. The mutation queue property controls how many writes that Accumulo will buffer in memory before performing a flush; this property should be set relative to the amount of JVM heap the TabletServers are given. Please note that the `table.durability` and `tserver.total.mutation.queue.max` properties only exists for >=Accumulo-1.7. There are no concise replacements for these properties in earlier versions. accumulo> config -s table.durability=flush accumulo> config -s tserver.total.mutation.queue.max=256M accumulo> config -t usertable -s table.file.compress.type=snappy On repeated data loads, the following commands may be helpful to re-set the state of the table quickly. accumulo> createtable tmp --copy-splits usertable --copy-config usertable accumulo> deletetable --force usertable accumulo> renametable tmp usertable accumulo> compact --wait -t accumulo.metadata ### 4. Load Data and Run Tests Load the data: ./bin/ycsb load accumulo1.8 -s -P workloads/workloada \ -p accumulo.zooKeepers=localhost \ -p accumulo.columnFamily=ycsb \ -p accumulo.instanceName=ycsb \ -p accumulo.username=user \ -p accumulo.password=supersecret \ > outputLoad.txt Run the workload test: ./bin/ycsb run accumulo1.8 -s -P workloads/workloada \ -p accumulo.zooKeepers=localhost \ -p accumulo.columnFamily=ycsb \ -p accumulo.instanceName=ycsb \ -p accumulo.username=user \ -p accumulo.password=supersecret \ > outputLoad.txt ## Accumulo Configuration Parameters - `accumulo.zooKeepers` - The Accumulo cluster's [zookeeper servers](https://accumulo.apache.org/1.8/accumulo_user_manual.html#_connecting). - Should contain a comma separated list of of hostname or hostname:port values. - No default value. - `accumulo.columnFamily` - The name of the column family to use to store the data within the table. - No default value. - `accumulo.instanceName` - Name of the Accumulo [instance](https://accumulo.apache.org/1.8/accumulo_user_manual.html#_connecting). - No default value. - `accumulo.username` - The username to use when connecting to Accumulo. - No default value. - `accumulo.password` - The password for the user connecting to Accumulo. - No default value. ================================================ FILE: accumulo1.8/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent accumulo1.8-binding Accumulo 1.8 DB Binding 2.6.4 true org.apache.accumulo accumulo-core ${accumulo.1.8.version} org.apache.hadoop hadoop-common ${hadoop.version} jdk.tools jdk.tools com.yahoo.ycsb core ${project.version} provided junit junit 4.12 test org.apache.accumulo accumulo-minicluster ${accumulo.1.8.version} test org.slf4j slf4j-api 1.7.13 ../workloads workloads src/test/resources ================================================ FILE: accumulo1.8/src/main/conf/accumulo.properties ================================================ # Copyright 2014 Cloudera, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Sample Accumulo configuration properties # # You may either set properties here or via the command line. # # This will influence the keys we write accumulo.columnFamily=YCSB # This should be set based on your Accumulo cluster #accumulo.instanceName=ExampleInstance # Comma separated list of host:port tuples for the ZooKeeper quorum used # by your Accumulo cluster #accumulo.zooKeepers=zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181 # This user will need permissions on the table YCSB works against #accumulo.username=ycsb #accumulo.password=protectyaneck # Controls how long our client writer will wait to buffer more data # measured in milliseconds accumulo.batchWriterMaxLatency=30000 # Controls how much data our client will attempt to buffer before sending # measured in bytes accumulo.batchWriterSize=100000 # Controls how many worker threads our client will use to parallelize writes accumulo.batchWriterThreads=1 ================================================ FILE: accumulo1.8/src/main/java/com/yahoo/ycsb/db/accumulo/AccumuloClient.java ================================================ /** * Copyright (c) 2011 YCSB++ project, 2014-2016 YCSB contributors. * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.accumulo; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.SortedMap; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CleanUp; import org.apache.hadoop.io.Text; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; /** * Accumulo binding for YCSB. */ public class AccumuloClient extends DB { private ZooKeeperInstance inst; private Connector connector; private Text colFam = new Text(""); private byte[] colFamBytes = new byte[0]; private final ConcurrentHashMap writers = new ConcurrentHashMap<>(); static { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { CleanUp.shutdownNow(); } }); } @Override public void init() throws DBException { colFam = new Text(getProperties().getProperty("accumulo.columnFamily")); colFamBytes = colFam.toString().getBytes(UTF_8); inst = new ZooKeeperInstance(new ClientConfiguration() .withInstance(getProperties().getProperty("accumulo.instanceName")) .withZkHosts(getProperties().getProperty("accumulo.zooKeepers"))); try { String principal = getProperties().getProperty("accumulo.username"); AuthenticationToken token = new PasswordToken(getProperties().getProperty("accumulo.password")); connector = inst.getConnector(principal, token); } catch (AccumuloException | AccumuloSecurityException e) { throw new DBException(e); } if (!(getProperties().getProperty("accumulo.pcFlag", "none").equals("none"))) { System.err.println("Sorry, the ZK based producer/consumer implementation has been removed. " + "Please see YCSB issue #416 for work on adding a general solution to coordinated work."); } } @Override public void cleanup() throws DBException { try { Iterator iterator = writers.values().iterator(); while (iterator.hasNext()) { BatchWriter writer = iterator.next(); writer.close(); iterator.remove(); } } catch (MutationsRejectedException e) { throw new DBException(e); } } /** * Called when the user specifies a table that isn't the same as the existing * table. Connect to it and if necessary, close our current connection. * * @param table * The table to open. */ public BatchWriter getWriter(String table) throws TableNotFoundException { // tl;dr We're paying a cost for the ConcurrentHashMap here to deal with the DB api. // We know that YCSB is really only ever going to send us data for one table, so using // a concurrent data structure is overkill (especially in such a hot code path). // However, the impact seems to be relatively negligible in trivial local tests and it's // "more correct" WRT to the API. BatchWriter writer = writers.get(table); if (null == writer) { BatchWriter newWriter = createBatchWriter(table); BatchWriter oldWriter = writers.putIfAbsent(table, newWriter); // Someone beat us to creating a BatchWriter for this table, use their BatchWriters if (null != oldWriter) { try { // Make sure to clean up our new batchwriter! newWriter.close(); } catch (MutationsRejectedException e) { throw new RuntimeException(e); } writer = oldWriter; } else { writer = newWriter; } } return writer; } /** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); } /** * Gets a scanner from Accumulo over one row. * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(String table, Text row, Set fields) throws TableNotFoundException { Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(row)); if (fields != null) { for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } return scanner; } @Override public Status read(String table, String key, Set fields, Map result) { Scanner scanner = null; try { scanner = getRow(table, new Text(key), null); // Pick out the results we care about. final Text cq = new Text(); for (Entry entry : scanner) { entry.getKey().getColumnQualifier(cq); Value v = entry.getValue(); byte[] buf = v.get(); result.put(cq.toString(), new ByteArrayByteIterator(buf)); } } catch (Exception e) { System.err.println("Error trying to reading Accumulo table " + table + " " + key); e.printStackTrace(); return Status.ERROR; } finally { if (null != scanner) { scanner.close(); } } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // Just make the end 'infinity' and only read as much as we need. Scanner scanner = null; try { scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(new Text(startkey), null)); // Have Accumulo send us complete rows, serialized in a single Key-Value pair IteratorSetting cfg = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(cfg); // If no fields are provided, we assume one column/row. if (fields != null) { // And add each of them as fields we want. for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } int count = 0; for (Entry entry : scanner) { // Deserialize the row SortedMap row = WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()); HashMap rowData; if (null != fields) { rowData = new HashMap<>(fields.size()); } else { rowData = new HashMap<>(); } result.add(rowData); // Parse the data in the row, avoid unnecessary Text object creation final Text cq = new Text(); for (Entry rowEntry : row.entrySet()) { rowEntry.getKey().getColumnQualifier(cq); rowData.put(cq.toString(), new ByteArrayByteIterator(rowEntry.getValue().get())); } if (count++ == recordcount) { // Done reading the last row. break; } } } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table."); e.printStackTrace(); return Status.ERROR; } catch (IOException e) { System.err.println("Error deserializing data from Accumulo."); e.printStackTrace(); return Status.ERROR; } finally { if (null != scanner) { scanner.close(); } } return Status.OK; } @Override public Status update(String table, String key, Map values) { BatchWriter bw = null; try { bw = getWriter(table); } catch (TableNotFoundException e) { System.err.println("Error opening batch writer to Accumulo table " + table); e.printStackTrace(); return Status.ERROR; } Mutation mutInsert = new Mutation(key.getBytes(UTF_8)); for (Map.Entry entry : values.entrySet()) { mutInsert.put(colFamBytes, entry.getKey().getBytes(UTF_8), entry.getValue().toArray()); } try { bw.addMutation(mutInsert); } catch (MutationsRejectedException e) { System.err.println("Error performing update."); e.printStackTrace(); return Status.ERROR; } return Status.BATCHED_OK; } @Override public Status insert(String t, String key, Map values) { return update(t, key, values); } @Override public Status delete(String table, String key) { BatchWriter bw; try { bw = getWriter(table); } catch (TableNotFoundException e) { System.err.println("Error trying to connect to Accumulo table."); e.printStackTrace(); return Status.ERROR; } try { deleteRow(table, new Text(key), bw); } catch (TableNotFoundException | MutationsRejectedException e) { System.err.println("Error performing delete."); e.printStackTrace(); return Status.ERROR; } catch (RuntimeException e) { System.err.println("Error performing delete."); e.printStackTrace(); return Status.ERROR; } return Status.OK; } // These functions are adapted from RowOperations.java: private void deleteRow(String table, Text row, BatchWriter bw) throws MutationsRejectedException, TableNotFoundException { // TODO Use a batchDeleter instead deleteRow(getRow(table, row, null), bw); } /** * Deletes a row, given a Scanner of JUST that row. */ private void deleteRow(Scanner scanner, BatchWriter bw) throws MutationsRejectedException { Mutation deleter = null; // iterate through the keys final Text row = new Text(); final Text cf = new Text(); final Text cq = new Text(); for (Entry entry : scanner) { // create a mutation for the row if (deleter == null) { entry.getKey().getRow(row); deleter = new Mutation(row); } entry.getKey().getColumnFamily(cf); entry.getKey().getColumnQualifier(cq); // the remove function adds the key with the delete flag set to true deleter.putDelete(cf, cq); } bw.addMutation(deleter); } } ================================================ FILE: accumulo1.8/src/main/java/com/yahoo/ycsb/db/accumulo/package-info.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for Apache Accumulo. */ package com.yahoo.ycsb.db.accumulo; ================================================ FILE: accumulo1.8/src/test/java/com/yahoo/ycsb/db/accumulo/AccumuloTest.java ================================================ /* * Copyright (c) 2016 YCSB contributors. * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.accumulo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.util.Map.Entry; import java.util.Properties; import com.yahoo.ycsb.Workload; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.workloads.CoreWorkload; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Use an Accumulo MiniCluster to test out basic workload operations with * the Accumulo binding. */ public class AccumuloTest { private static final Logger LOG = LoggerFactory.getLogger(AccumuloTest.class); private static final int INSERT_COUNT = 2000; private static final int TRANSACTION_COUNT = 2000; @ClassRule public static TemporaryFolder workingDir = new TemporaryFolder(); @Rule public TestName test = new TestName(); private static MiniAccumuloCluster cluster; private static Properties properties; private Workload workload; private DB client; private Properties workloadProps; private static boolean isWindows() { final String os = System.getProperty("os.name"); return os.startsWith("Windows"); } @BeforeClass public static void setup() throws Exception { // Minicluster setup fails on Windows with an UnsatisfiedLinkError. // Skip if windows. assumeTrue(!isWindows()); cluster = new MiniAccumuloCluster(workingDir.newFolder("accumulo").getAbsoluteFile(), "protectyaneck"); LOG.debug("starting minicluster"); cluster.start(); LOG.debug("creating connection for admin operations."); // set up the table and user final Connector admin = cluster.getConnector("root", "protectyaneck"); admin.tableOperations().create(CoreWorkload.TABLENAME_PROPERTY_DEFAULT); admin.securityOperations().createLocalUser("ycsb", new PasswordToken("protectyaneck")); admin.securityOperations().grantTablePermission("ycsb", CoreWorkload.TABLENAME_PROPERTY_DEFAULT, TablePermission.READ); admin.securityOperations().grantTablePermission("ycsb", CoreWorkload.TABLENAME_PROPERTY_DEFAULT, TablePermission.WRITE); // set properties the binding will read properties = new Properties(); properties.setProperty("accumulo.zooKeepers", cluster.getZooKeepers()); properties.setProperty("accumulo.instanceName", cluster.getInstanceName()); properties.setProperty("accumulo.columnFamily", "family"); properties.setProperty("accumulo.username", "ycsb"); properties.setProperty("accumulo.password", "protectyaneck"); // cut down the batch writer timeout so that writes will push through. properties.setProperty("accumulo.batchWriterMaxLatency", "4"); // set these explicitly to the defaults at the time we're compiled, since they'll be inlined in our class. properties.setProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT); properties.setProperty(CoreWorkload.FIELD_COUNT_PROPERTY, CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT); properties.setProperty(CoreWorkload.INSERT_ORDER_PROPERTY, "ordered"); } @AfterClass public static void clusterCleanup() throws Exception { if (cluster != null) { LOG.debug("shutting down minicluster"); cluster.stop(); cluster = null; } } @Before public void client() throws Exception { LOG.debug("Loading workload properties for {}", test.getMethodName()); workloadProps = new Properties(); workloadProps.load(getClass().getResourceAsStream("/workloads/" + test.getMethodName())); for (String prop : properties.stringPropertyNames()) { workloadProps.setProperty(prop, properties.getProperty(prop)); } // TODO we need a better test rig for 'run this ycsb workload' LOG.debug("initializing measurements and workload"); Measurements.setProperties(workloadProps); workload = new CoreWorkload(); workload.init(workloadProps); LOG.debug("initializing client"); client = new AccumuloClient(); client.setProperties(workloadProps); client.init(); } @After public void cleanup() throws Exception { if (client != null) { LOG.debug("cleaning up client"); client.cleanup(); client = null; } if (workload != null) { LOG.debug("cleaning up workload"); workload.cleanup(); } } @After public void truncateTable() throws Exception { if (cluster != null) { LOG.debug("truncating table {}", CoreWorkload.TABLENAME_PROPERTY_DEFAULT); final Connector admin = cluster.getConnector("root", "protectyaneck"); admin.tableOperations().deleteRows(CoreWorkload.TABLENAME_PROPERTY_DEFAULT, null, null); } } @Test public void workloada() throws Exception { runWorkload(); } @Test public void workloadb() throws Exception { runWorkload(); } @Test public void workloadc() throws Exception { runWorkload(); } @Test public void workloadd() throws Exception { runWorkload(); } @Test public void workloade() throws Exception { runWorkload(); } /** * go through a workload cycle. *
    *
  1. initialize thread-specific state *
  2. load the workload dataset *
  3. run workload transactions *
*/ private void runWorkload() throws Exception { final Object state = workload.initThread(workloadProps,0,0); LOG.debug("load"); for (int i = 0; i < INSERT_COUNT; i++) { assertTrue("insert failed.", workload.doInsert(client, state)); } // Ensure we wait long enough for the batch writer to flush // TODO accumulo client should be flushing per insert by default. Thread.sleep(2000); LOG.debug("verify number of cells"); final Scanner scanner = cluster.getConnector("root", "protectyaneck").createScanner(CoreWorkload.TABLENAME_PROPERTY_DEFAULT, Authorizations.EMPTY); int count = 0; for (Entry entry : scanner) { count++; } assertEquals("Didn't get enough total cells.", (Integer.valueOf(CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT) * INSERT_COUNT), count); LOG.debug("run"); for (int i = 0; i < TRANSACTION_COUNT; i++) { assertTrue("transaction failed.", workload.doTransaction(client, state)); } } } ================================================ FILE: accumulo1.8/src/test/resources/log4j.properties ================================================ # # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=INFO, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZooKeeper log4j.logger.com.yahoo.ycsb.db.accumulo=DEBUG log4j.logger.org.apache.zookeeper=ERROR log4j.logger.org.apache.accumulo=WARN ================================================ FILE: aerospike/README.md ================================================ ## Quick Start This section describes how to run YCSB on Aerospike. ### 1. Start Aerospike ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:aerospike-binding -am clean package ### 4. Provide Aerospike Connection Parameters The following connection parameters are available. * `as.host` - The Aerospike cluster to connect to (default: `localhost`) * `as.port` - The port to connect to (default: `3000`) * `as.user` - The user to connect as (no default) * `as.password` - The password for the user (no default) * `as.timeout` - The transaction and connection timeout (in ms, default: `10000`) * `as.namespace` - The namespace to be used for the benchmark (default: `ycsb`) Add them to the workload or set them with the shell command, as in: ./bin/ycsb load aerospike -s -P workloads/workloada -p as.timeout=5000 >outputLoad.txt ### 5. Load Data and Run Tests Load the data: ./bin/ycsb load aerospike -s -P workloads/workloada >outputLoad.txt Run the workload test: ./bin/ycsb run aerospike -s -P workloads/workloada >outputRun.txt ================================================ FILE: aerospike/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent aerospike-binding Aerospike DB Binding jar com.aerospike aerospike-client ${aerospike.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.aerospike.client.AerospikeException; import com.aerospike.client.Bin; import com.aerospike.client.Key; import com.aerospike.client.Record; import com.aerospike.client.policy.ClientPolicy; import com.aerospike.client.policy.Policy; import com.aerospike.client.policy.RecordExistsAction; import com.aerospike.client.policy.WritePolicy; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * YCSB binding for Areospike. */ public class AerospikeClient extends com.yahoo.ycsb.DB { private static final String DEFAULT_HOST = "localhost"; private static final String DEFAULT_PORT = "3000"; private static final String DEFAULT_TIMEOUT = "10000"; private static final String DEFAULT_NAMESPACE = "ycsb"; private String namespace = null; private com.aerospike.client.AerospikeClient client = null; private Policy readPolicy = new Policy(); private WritePolicy insertPolicy = new WritePolicy(); private WritePolicy updatePolicy = new WritePolicy(); private WritePolicy deletePolicy = new WritePolicy(); @Override public void init() throws DBException { insertPolicy.recordExistsAction = RecordExistsAction.CREATE_ONLY; updatePolicy.recordExistsAction = RecordExistsAction.REPLACE_ONLY; Properties props = getProperties(); namespace = props.getProperty("as.namespace", DEFAULT_NAMESPACE); String host = props.getProperty("as.host", DEFAULT_HOST); String user = props.getProperty("as.user"); String password = props.getProperty("as.password"); int port = Integer.parseInt(props.getProperty("as.port", DEFAULT_PORT)); int timeout = Integer.parseInt(props.getProperty("as.timeout", DEFAULT_TIMEOUT)); readPolicy.timeout = timeout; insertPolicy.timeout = timeout; updatePolicy.timeout = timeout; deletePolicy.timeout = timeout; ClientPolicy clientPolicy = new ClientPolicy(); if (user != null && password != null) { clientPolicy.user = user; clientPolicy.password = password; } try { client = new com.aerospike.client.AerospikeClient(clientPolicy, host, port); } catch (AerospikeException e) { throw new DBException(String.format("Error while creating Aerospike " + "client for %s:%d.", host, port), e); } } @Override public void cleanup() throws DBException { client.close(); } @Override public Status read(String table, String key, Set fields, Map result) { try { Record record; if (fields != null) { record = client.get(readPolicy, new Key(namespace, table, key), fields.toArray(new String[fields.size()])); } else { record = client.get(readPolicy, new Key(namespace, table, key)); } if (record == null) { System.err.println("Record key " + key + " not found (read)"); return Status.ERROR; } for (Map.Entry entry: record.bins.entrySet()) { result.put(entry.getKey(), new ByteArrayByteIterator((byte[])entry.getValue())); } return Status.OK; } catch (AerospikeException e) { System.err.println("Error while reading key " + key + ": " + e); return Status.ERROR; } } @Override public Status scan(String table, String start, int count, Set fields, Vector> result) { System.err.println("Scan not implemented"); return Status.ERROR; } private Status write(String table, String key, WritePolicy writePolicy, Map values) { Bin[] bins = new Bin[values.size()]; int index = 0; for (Map.Entry entry: values.entrySet()) { bins[index] = new Bin(entry.getKey(), entry.getValue().toArray()); ++index; } Key keyObj = new Key(namespace, table, key); try { client.put(writePolicy, keyObj, bins); return Status.OK; } catch (AerospikeException e) { System.err.println("Error while writing key " + key + ": " + e); return Status.ERROR; } } @Override public Status update(String table, String key, Map values) { return write(table, key, updatePolicy, values); } @Override public Status insert(String table, String key, Map values) { return write(table, key, insertPolicy, values); } @Override public Status delete(String table, String key) { try { if (!client.delete(deletePolicy, new Key(namespace, table, key))) { System.err.println("Record key " + key + " not found (delete)"); return Status.ERROR; } return Status.OK; } catch (AerospikeException e) { System.err.println("Error while deleting key " + key + ": " + e); return Status.ERROR; } } } ================================================ FILE: aerospike/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for Areospike. */ package com.yahoo.ycsb.db; ================================================ FILE: arangodb/.gitignore ================================================ /bin/ ================================================ FILE: arangodb/README.md ================================================ ## Quick Start This section describes how to run YCSB on ArangoDB. ### 1. Start ArangoDB See https://docs.arangodb.com/Installing/index.html ### 2. Install Java and Maven Go to http://www.oracle.com/technetwork/java/javase/downloads/index.html and get the url to download the rpm into your server. For example: wget http://download.oracle.com/otn-pub/java/jdk/7u40-b43/jdk-7u40-linux-x64.rpm?AuthParam=11232426132 -o jdk-7u40-linux-x64.rpm rpm -Uvh jdk-7u40-linux-x64.rpm Or install via yum/apt-get sudo yum install java-devel Download MVN from http://maven.apache.org/download.cgi wget http://ftp.heanet.ie/mirrors/www.apache.org/dist/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz sudo tar xzf apache-maven-*-bin.tar.gz -C /usr/local cd /usr/local sudo ln -s apache-maven-* maven sudo vi /etc/profile.d/maven.sh Add the following to `maven.sh` export M2_HOME=/usr/local/maven export PATH=${M2_HOME}/bin:${PATH} Reload bash and test mvn bash mvn -version ### 3. Set Up YCSB Clone this YCSB source code: git clone https://github.com/brianfrankcooper/YCSB.git ### 4. Run YCSB Now you are ready to run! First, drop the existing collection: "usertable" under database "ycsb": db._collection("usertable").drop() Then, load the data: ./bin/ycsb load arangodb -s -P workloads/workloada -p arangodb.ip=xxx -p arangodb.port=xxx Then, run the workload: ./bin/ycsb run arangodb -s -P workloads/workloada -p arangodb.ip=xxx -p arangodb.port=xxx See the next section for the list of configuration parameters for ArangoDB. ## ArangoDB Configuration Parameters - `arangodb.ip` - Default value is `localhost` - `arangodb.port` - Default value is `8529`. - `arangodb.waitForSync` - Default value is `true`. - `arangodb.transactionUpdate` - Default value is `false`. - `arangodb.dropDBBeforeRun` - Default value is `false`. ================================================ FILE: arangodb/conf/logback.xml ================================================ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n ================================================ FILE: arangodb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent arangodb-binding ArangoDB Binding jar com.arangodb arangodb-java-driver ${arangodb.version} com.yahoo.ycsb core ${project.version} provided org.slf4j slf4j-api 1.7.13 jar compile ch.qos.logback logback-classic 1.1.3 jar provided ch.qos.logback logback-core 1.1.3 jar provided junit junit 4.12 test ================================================ FILE: arangodb/src/main/java/com/yahoo/ycsb/db/ArangoDBClient.java ================================================ /** * Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.arangodb.ArangoConfigure; import com.arangodb.ArangoDriver; import com.arangodb.ArangoException; import com.arangodb.ArangoHost; import com.arangodb.DocumentCursor; import com.arangodb.ErrorNums; import com.arangodb.entity.BaseDocument; import com.arangodb.entity.DocumentEntity; import com.arangodb.entity.EntityFactory; import com.arangodb.entity.TransactionEntity; import com.arangodb.util.MapBuilder; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.StringByteIterator; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * ArangoDB binding for YCSB framework using the ArangoDB Inc. driver *

* See the README.md for configuration information. *

* * @see ArangoDB Inc. * driver */ public class ArangoDBClient extends DB { private static Logger logger = LoggerFactory.getLogger(ArangoDBClient.class); /** * The database name to access. */ private static String databaseName = "ycsb"; /** * Count the number of times initialized to teardown on the last * {@link #cleanup()}. */ private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); /** ArangoDB Driver related, Singleton. */ private static ArangoDriver arangoDriver; private static Boolean dropDBBeforeRun; private static Boolean waitForSync = true; private static Boolean transactionUpdate = false; /** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. * * Actually, one client process will share one DB instance here.(Coincide to * mongoDB driver) */ @Override public void init() throws DBException { INIT_COUNT.incrementAndGet(); synchronized (ArangoDBClient.class) { if (arangoDriver != null) { return; } Properties props = getProperties(); // Set the DB address String ip = props.getProperty("arangodb.ip", "localhost"); String portStr = props.getProperty("arangodb.port", "8529"); int port = Integer.parseInt(portStr); // If clear db before run String dropDBBeforeRunStr = props.getProperty("arangodb.dropDBBeforeRun", "false"); dropDBBeforeRun = Boolean.parseBoolean(dropDBBeforeRunStr); // Set the sync mode String waitForSyncStr = props.getProperty("arangodb.waitForSync", "false"); waitForSync = Boolean.parseBoolean(waitForSyncStr); // Set if transaction for update String transactionUpdateStr = props.getProperty("arangodb.transactionUpdate", "false"); transactionUpdate = Boolean.parseBoolean(transactionUpdateStr); // Init ArangoDB connection try { ArangoConfigure arangoConfigure = new ArangoConfigure(); arangoConfigure.setArangoHost(new ArangoHost(ip, port)); arangoConfigure.init(); arangoDriver = new ArangoDriver(arangoConfigure); } catch (Exception e) { logger.error("Failed to initialize ArangoDB", e); System.exit(-1); } // Init the database if (dropDBBeforeRun) { // Try delete first try { arangoDriver.deleteDatabase(databaseName); } catch (ArangoException e) { if (e.getErrorNumber() != ErrorNums.ERROR_ARANGO_DATABASE_NOT_FOUND) { logger.error("Failed to delete database: {} with ex: {}", databaseName, e.toString()); System.exit(-1); } else { logger.info("Fail to delete DB, already deleted: {}", databaseName); } } } try { arangoDriver.createDatabase(databaseName); logger.info("Database created: " + databaseName); } catch (ArangoException e) { if (e.getErrorNumber() != ErrorNums.ERROR_ARANGO_DUPLICATE_NAME) { logger.error("Failed to create database: {} with ex: {}", databaseName, e.toString()); System.exit(-1); } else { logger.info("DB already exists: {}", databaseName); } } // Always set the default db arangoDriver.setDefaultDatabase(databaseName); logger.info("ArangoDB client connection created to {}:{}", ip, port); // Log the configuration logger.info("Arango Configuration: dropDBBeforeRun: {}; address: {}:{}; databaseName: {};" + " waitForSync: {}; transactionUpdate: {};", dropDBBeforeRun, ip, port, databaseName, waitForSync, transactionUpdate); } } /** * Cleanup any state for this DB. Called once per DB instance; there is one * DB instance per client thread. * * Actually, one client process will share one DB instance here.(Coincide to * mongoDB driver) */ @Override public void cleanup() throws DBException { if (INIT_COUNT.decrementAndGet() == 0) { arangoDriver = null; logger.info("Local cleaned up."); } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status insert(String table, String key, Map values) { try { BaseDocument toInsert = new BaseDocument(key); for (Map.Entry entry : values.entrySet()) { toInsert.addAttribute(entry.getKey(), byteIteratorToString(entry.getValue())); } arangoDriver.createDocument(table, toInsert, true/*create collection if not exist*/, waitForSync); return Status.OK; } catch (ArangoException e) { if (e.getErrorNumber() != ErrorNums.ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED) { logger.error("Fail to insert: {} {} with ex {}", table, key, e.toString()); } else { logger.debug("Trying to create document with duplicate key: {} {}", table, key); return Status.BAD_REQUEST; } } catch (RuntimeException e) { logger.error("Exception while trying insert {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Read a record from the database. Each field/value pair from the result * will be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @SuppressWarnings("unchecked") @Override public Status read(String table, String key, Set fields, Map result) { try { DocumentEntity targetDoc = arangoDriver.getDocument(table, key, BaseDocument.class); BaseDocument aDocument = targetDoc.getEntity(); if (!this.fillMap(result, aDocument.getProperties(), fields)) { return Status.ERROR; } return Status.OK; } catch (ArangoException e) { if (e.getErrorNumber() != ErrorNums.ERROR_ARANGO_DOCUMENT_NOT_FOUND) { logger.error("Fail to read: {} {} with ex {}", table, key, e.toString()); } else { logger.debug("Trying to read document not exist: {} {}", table, key); return Status.NOT_FOUND; } } catch (RuntimeException e) { logger.error("Exception while trying read {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public Status update(String table, String key, Map values) { try { if (!transactionUpdate) { BaseDocument updateDoc = new BaseDocument(); for (String field : values.keySet()) { updateDoc.addAttribute(field, byteIteratorToString(values.get(field))); } arangoDriver.updateDocument(table, key, updateDoc); return Status.OK; } else { // id for documentHandle String transactionAction = "function (id) {" // use internal database functions + "var db = require('internal').db;" // collection.update(document, data, overwrite, keepNull, waitForSync) + String.format("db._update(id, %s, true, false, %s);}", mapToJson(values), Boolean.toString(waitForSync).toLowerCase()); TransactionEntity transaction = arangoDriver.createTransaction(transactionAction); transaction.addWriteCollection(table); transaction.setParams(createDocumentHandle(table, key)); arangoDriver.executeTransaction(transaction); return Status.OK; } } catch (ArangoException e) { if (e.getErrorNumber() != ErrorNums.ERROR_ARANGO_DOCUMENT_NOT_FOUND) { logger.error("Fail to update: {} {} with ex {}", table, key, e.toString()); } else { logger.debug("Trying to update document not exist: {} {}", table, key); return Status.NOT_FOUND; } } catch (RuntimeException e) { logger.error("Exception while trying update {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status delete(String table, String key) { try { arangoDriver.deleteDocument(table, key); return Status.OK; } catch (ArangoException e) { if (e.getErrorNumber() != ErrorNums.ERROR_ARANGO_DOCUMENT_NOT_FOUND) { logger.error("Fail to delete: {} {} with ex {}", table, key, e.toString()); } else { logger.debug("Trying to delete document not exist: {} {}", table, key); return Status.NOT_FOUND; } } catch (RuntimeException e) { logger.error("Exception while trying delete {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Perform a range scan for a set of records in the database. Each * field/value pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { DocumentCursor cursor = null; try { String aqlQuery = String.format( "FOR target IN %s FILTER target._key >= @key SORT target._key ASC LIMIT %d RETURN %s ", table, recordcount, constructReturnForAQL(fields, "target")); Map bindVars = new MapBuilder().put("key", startkey).get(); cursor = arangoDriver.executeDocumentQuery(aqlQuery, bindVars, null, BaseDocument.class); Iterator iterator = cursor.entityIterator(); while (iterator.hasNext()) { BaseDocument aDocument = iterator.next(); HashMap aMap = new HashMap(aDocument.getProperties().size()); if (!this.fillMap(aMap, aDocument.getProperties())) { return Status.ERROR; } result.add(aMap); } return Status.OK; } catch (Exception e) { logger.error("Exception while trying scan {} {} {} with ex {}", table, startkey, recordcount, e.toString()); } finally { if (cursor != null) { try { cursor.close(); } catch (ArangoException e) { logger.error("Fail to close cursor", e); } } } return Status.ERROR; } private String createDocumentHandle(String collectionName, String documentKey) throws ArangoException { validateCollectionName(collectionName); return collectionName + "/" + documentKey; } private void validateCollectionName(String name) throws ArangoException { if (name.indexOf('/') != -1) { throw new ArangoException("does not allow '/' in name."); } } private String constructReturnForAQL(Set fields, String targetName) { // Construct the AQL query string. String resultDes = targetName; if (fields != null && fields.size() != 0) { StringBuilder builder = new StringBuilder("{"); for (String field : fields) { builder.append(String.format("\n\"%s\" : %s.%s,", field, targetName, field)); } //Replace last ',' to newline. builder.setCharAt(builder.length() - 1, '\n'); builder.append("}"); resultDes = builder.toString(); } return resultDes; } private boolean fillMap(Map resultMap, Map properties) { return fillMap(resultMap, properties, null); } /** * Fills the map with the properties from the BaseDocument. * * @param resultMap * The map to fill/ * @param obj * The object to copy values from. * @return isSuccess */ @SuppressWarnings("unchecked") private boolean fillMap(Map resultMap, Map properties, Set fields) { if (fields == null || fields.size() == 0) { for (Map.Entry entry : properties.entrySet()) { if (entry.getValue() instanceof String) { resultMap.put(entry.getKey(), stringToByteIterator((String)(entry.getValue()))); } else { logger.error("Error! Not the format expected! Actually is {}", entry.getValue().getClass().getName()); return false; } } } else { for (String field : fields) { if (properties.get(field) instanceof String) { resultMap.put(field, stringToByteIterator((String)(properties.get(field)))); } else { logger.error("Error! Not the format expected! Actually is {}", properties.get(field).getClass().getName()); return false; } } } return true; } private String byteIteratorToString(ByteIterator byteIter) { return new String(byteIter.toArray()); } private ByteIterator stringToByteIterator(String content) { return new StringByteIterator(content); } private String mapToJson(Map values) { Map intervalRst = new HashMap(); for (Map.Entry entry : values.entrySet()) { intervalRst.put(entry.getKey(), byteIteratorToString(entry.getValue())); } return EntityFactory.toJsonString(intervalRst); } } ================================================ FILE: arangodb/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for ArangoDB. */ package com.yahoo.ycsb.db; ================================================ FILE: arangodb3/.gitignore ================================================ ================================================ FILE: arangodb3/README.md ================================================ ## Quick Start This section describes how to run YCSB on ArangoDB. ### 1. Start ArangoDB See https://docs.arangodb.com/Installing/index.html ### 2. Install Java and Maven Go to http://www.oracle.com/technetwork/java/javase/downloads/index.html and get the url to download the rpm into your server. For example: wget http://download.oracle.com/otn-pub/java/jdk/7u40-b43/jdk-7u40-linux-x64.rpm?AuthParam=11232426132 -o jdk-7u40-linux-x64.rpm rpm -Uvh jdk-7u40-linux-x64.rpm Or install via yum/apt-get sudo yum install java-devel Download MVN from http://maven.apache.org/download.cgi wget http://ftp.heanet.ie/mirrors/www.apache.org/dist/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz sudo tar xzf apache-maven-*-bin.tar.gz -C /usr/local cd /usr/local sudo ln -s apache-maven-* maven sudo vi /etc/profile.d/maven.sh Add the following to `maven.sh` export M2_HOME=/usr/local/maven export PATH=${M2_HOME}/bin:${PATH} Reload bash and test mvn bash mvn -version ### 3. Set Up YCSB Clone this YCSB source code: git clone https://github.com/brianfrankcooper/YCSB.git ### 4. Run YCSB Now you are ready to run! First, drop the existing collection: "usertable" under database "ycsb": db._collection("usertable").drop() Then, load the data: ./bin/ycsb load arangodb3 -s -P workloads/workloada -p arangodb.ip=xxx -p arangodb.port=xxx Then, run the workload: ./bin/ycsb run arangodb3 -s -P workloads/workloada -p arangodb.ip=xxx -p arangodb.port=xxx See the next section for the list of configuration parameters for ArangoDB. ## ArangoDB Configuration Parameters - `arangodb.ip` - Default value is `localhost` - `arangodb.port` - Default value is `8529`. - `arangodb.waitForSync` - Default value is `true`. - `arangodb.transactionUpdate` - Default value is `false`. - `arangodb.dropDBBeforeRun` - Default value is `false`. ================================================ FILE: arangodb3/conf/logback.xml ================================================ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n ================================================ FILE: arangodb3/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent arangodb3-binding ArangoDB3 Binding jar com.arangodb arangodb-java-driver ${arangodb3.version} com.yahoo.ycsb core ${project.version} provided org.slf4j slf4j-api 1.7.13 jar compile ch.qos.logback logback-classic 1.1.3 jar provided ch.qos.logback logback-core 1.1.3 jar provided junit junit 4.12 test ================================================ FILE: arangodb3/src/main/java/com/yahoo/ycsb/db/arangodb/ArangoDB3Client.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.arangodb; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.arangodb.ArangoCursor; import com.arangodb.ArangoDB; import com.arangodb.ArangoDBException; import com.arangodb.entity.BaseDocument; import com.arangodb.model.DocumentCreateOptions; import com.arangodb.model.TransactionOptions; import com.arangodb.util.MapBuilder; import com.arangodb.velocypack.VPackBuilder; import com.arangodb.velocypack.VPackSlice; import com.arangodb.velocypack.ValueType; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; /** * ArangoDB binding for YCSB framework using the ArangoDB Inc. driver *

* See the README.md for configuration information. *

* * @see ArangoDB Inc. * driver */ public class ArangoDB3Client extends DB { private static Logger logger = LoggerFactory.getLogger(ArangoDB3Client.class); /** * Count the number of times initialized to teardown on the last * {@link #cleanup()}. */ private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); /** ArangoDB Driver related, Singleton. */ private ArangoDB arangoDB; private String databaseName = "ycsb"; private String collectionName; private Boolean dropDBBeforeRun; private Boolean waitForSync = false; private Boolean transactionUpdate = false; /** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. * * Actually, one client process will share one DB instance here.(Coincide to * mongoDB driver) */ @Override public void init() throws DBException { synchronized (ArangoDB3Client.class) { Properties props = getProperties(); collectionName = props.getProperty("table", "usertable"); // Set the DB address String ip = props.getProperty("arangodb.ip", "localhost"); String portStr = props.getProperty("arangodb.port", "8529"); int port = Integer.parseInt(portStr); // If clear db before run String dropDBBeforeRunStr = props.getProperty("arangodb.dropDBBeforeRun", "false"); dropDBBeforeRun = Boolean.parseBoolean(dropDBBeforeRunStr); // Set the sync mode String waitForSyncStr = props.getProperty("arangodb.waitForSync", "false"); waitForSync = Boolean.parseBoolean(waitForSyncStr); // Set if transaction for update String transactionUpdateStr = props.getProperty("arangodb.transactionUpdate", "false"); transactionUpdate = Boolean.parseBoolean(transactionUpdateStr); // Init ArangoDB connection try { arangoDB = new ArangoDB.Builder().host(ip).port(port).build(); } catch (Exception e) { logger.error("Failed to initialize ArangoDB", e); System.exit(-1); } if(INIT_COUNT.getAndIncrement() == 0) { // Init the database if (dropDBBeforeRun) { // Try delete first try { arangoDB.db(databaseName).drop(); } catch (ArangoDBException e) { logger.info("Fail to delete DB: {}", databaseName); } } try { arangoDB.createDatabase(databaseName); logger.info("Database created: " + databaseName); } catch (ArangoDBException e) { logger.error("Failed to create database: {} with ex: {}", databaseName, e.toString()); } try { arangoDB.db(databaseName).createCollection(collectionName); logger.info("Collection created: " + collectionName); } catch (ArangoDBException e) { logger.error("Failed to create collection: {} with ex: {}", collectionName, e.toString()); } logger.info("ArangoDB client connection created to {}:{}", ip, port); // Log the configuration logger.info("Arango Configuration: dropDBBeforeRun: {}; address: {}:{}; databaseName: {};" + " waitForSync: {}; transactionUpdate: {};", dropDBBeforeRun, ip, port, databaseName, waitForSync, transactionUpdate); } } } /** * Cleanup any state for this DB. Called once per DB instance; there is one * DB instance per client thread. * * Actually, one client process will share one DB instance here.(Coincide to * mongoDB driver) */ @Override public void cleanup() throws DBException { if (INIT_COUNT.decrementAndGet() == 0) { arangoDB.shutdown(); arangoDB = null; logger.info("Local cleaned up."); } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status insert(String table, String key, Map values) { try { BaseDocument toInsert = new BaseDocument(key); for (Map.Entry entry : values.entrySet()) { toInsert.addAttribute(entry.getKey(), byteIteratorToString(entry.getValue())); } DocumentCreateOptions options = new DocumentCreateOptions().waitForSync(waitForSync); arangoDB.db(databaseName).collection(table).insertDocument(toInsert, options); return Status.OK; } catch (ArangoDBException e) { logger.error("Exception while trying insert {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Read a record from the database. Each field/value pair from the result * will be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public Status read(String table, String key, Set fields, Map result) { try { VPackSlice document = arangoDB.db(databaseName).collection(table).getDocument(key, VPackSlice.class, null); if (!this.fillMap(result, document, fields)) { return Status.ERROR; } return Status.OK; } catch (ArangoDBException e) { logger.error("Exception while trying read {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public Status update(String table, String key, Map values) { try { if (!transactionUpdate) { BaseDocument updateDoc = new BaseDocument(); for (Entry field : values.entrySet()) { updateDoc.addAttribute(field.getKey(), byteIteratorToString(field.getValue())); } arangoDB.db(databaseName).collection(table).updateDocument(key, updateDoc); return Status.OK; } else { // id for documentHandle String transactionAction = "function (id) {" // use internal database functions + "var db = require('internal').db;" // collection.update(document, data, overwrite, keepNull, waitForSync) + String.format("db._update(id, %s, true, false, %s);}", mapToJson(values), Boolean.toString(waitForSync).toLowerCase()); TransactionOptions options = new TransactionOptions(); options.writeCollections(table); options.params(createDocumentHandle(table, key)); arangoDB.db(databaseName).transaction(transactionAction, Void.class, options); return Status.OK; } } catch (ArangoDBException e) { logger.error("Exception while trying update {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status delete(String table, String key) { try { arangoDB.db(databaseName).collection(table).deleteDocument(key); return Status.OK; } catch (ArangoDBException e) { logger.error("Exception while trying delete {} {} with ex {}", table, key, e.toString()); } return Status.ERROR; } /** * Perform a range scan for a set of records in the database. Each * field/value pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { ArangoCursor cursor = null; try { String aqlQuery = String.format( "FOR target IN %s FILTER target._key >= @key SORT target._key ASC LIMIT %d RETURN %s ", table, recordcount, constructReturnForAQL(fields, "target")); Map bindVars = new MapBuilder().put("key", startkey).get(); cursor = arangoDB.db(databaseName).query(aqlQuery, bindVars, null, VPackSlice.class); while (cursor.hasNext()) { VPackSlice aDocument = cursor.next(); HashMap aMap = new HashMap(aDocument.size()); if (!this.fillMap(aMap, aDocument)) { return Status.ERROR; } result.add(aMap); } return Status.OK; } catch (Exception e) { logger.error("Exception while trying scan {} {} {} with ex {}", table, startkey, recordcount, e.toString()); } finally { if (cursor != null) { try { cursor.close(); } catch (IOException e) { logger.error("Fail to close cursor", e); } } } return Status.ERROR; } private String createDocumentHandle(String collection, String documentKey) throws ArangoDBException { validateCollectionName(collection); return collection + "/" + documentKey; } private void validateCollectionName(String name) throws ArangoDBException { if (name.indexOf('/') != -1) { throw new ArangoDBException("does not allow '/' in name."); } } private String constructReturnForAQL(Set fields, String targetName) { // Construct the AQL query string. String resultDes = targetName; if (fields != null && fields.size() != 0) { StringBuilder builder = new StringBuilder("{"); for (String field : fields) { builder.append(String.format("\n\"%s\" : %s.%s,", field, targetName, field)); } //Replace last ',' to newline. builder.setCharAt(builder.length() - 1, '\n'); builder.append("}"); resultDes = builder.toString(); } return resultDes; } private boolean fillMap(Map resultMap, VPackSlice document) { return fillMap(resultMap, document, null); } /** * Fills the map with the properties from the BaseDocument. * * @param resultMap * The map to fill/ * @param document * The record to read from * @param fields * The list of fields to read, or null for all of them * @return isSuccess */ private boolean fillMap(Map resultMap, VPackSlice document, Set fields) { if (fields == null || fields.size() == 0) { for (Iterator> iterator = document.objectIterator(); iterator.hasNext();) { Entry next = iterator.next(); VPackSlice value = next.getValue(); if (value.isString()) { resultMap.put(next.getKey(), stringToByteIterator(value.getAsString())); } else if (!value.isCustom()) { logger.error("Error! Not the format expected! Actually is {}", value.getClass().getName()); return false; } } } else { for (String field : fields) { VPackSlice value = document.get(field); if (value.isString()) { resultMap.put(field, stringToByteIterator(value.getAsString())); } else if (!value.isCustom()) { logger.error("Error! Not the format expected! Actually is {}", value.getClass().getName()); return false; } } } return true; } private String byteIteratorToString(ByteIterator byteIter) { return new String(byteIter.toArray()); } private ByteIterator stringToByteIterator(String content) { return new StringByteIterator(content); } private String mapToJson(Map values) { VPackBuilder builder = new VPackBuilder().add(ValueType.OBJECT); for (Map.Entry entry : values.entrySet()) { builder.add(entry.getKey(), byteIteratorToString(entry.getValue())); } builder.close(); return arangoDB.util().deserialize(builder.slice(), String.class); } } ================================================ FILE: arangodb3/src/main/java/com/yahoo/ycsb/db/arangodb/package-info.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for ArangoDB. */ package com.yahoo.ycsb.db.arangodb; ================================================ FILE: asynchbase/README.md ================================================ # AsyncHBase Driver for YCSB This driver provides a YCSB workload binding for Apache HBase using an alternative to the included HBase client. AsyncHBase is completely asynchronous for all operations and is particularly useful for write heavy workloads. Note that it supports a subset of the HBase client APIs but supports all public released versions of HBase. ## Quickstart ### 1. Setup Hbase Follow directions 1 to 3 from ``hbase098``'s readme. ### 2. Load a Workload Switch to the root of the YCSB repo and choose the workload you want to run and `load` it first. With the CLI you must provide the column family at a minimum if HBase is running on localhost. Otherwise you must provide connection properties via CLI or the path to a config file. Additional configuration parameters are available below. ``` bin/ycsb load asynchbase -p columnfamily=cf -P workloads/workloada ``` The `load` step only executes inserts into the datastore. After loading data, run the same workload to mix reads with writes. ``` bin/ycsb run asynchbase -p columnfamily=cf -P workloads/workloada ``` ## Configuration Options The following options can be configured using CLI (using the `-p` parameter) or via a JAVA style properties configuration file.. Check the [AsyncHBase Configuration](http://opentsdb.github.io/asynchbase/docs/build/html/configuration.html) project for additional tuning parameters. * `columnfamily`: (Required) The column family to target. * `config`: Optional full path to a configuration file with AsyncHBase options. * `hbase.zookeeper.quorum`: Zookeeper quorum list. * `hbase.zookeeper.znode.parent`: Path used by HBase in Zookeeper. Default is "/hbase". * `debug`: If true, prints debug information to standard out. The default is false. * `clientbuffering`: Whether or not to use client side buffering and batching of write operations. This can significantly improve performance and defaults to true. * `durable`: When set to false, writes and deletes bypass the WAL for quicker responses. Default is true. * `jointimeout`: A timeout value, in milliseconds, for waiting on operations synchronously before an error is thrown. * `prefetchmeta`: Whether or not to read meta for all regions in the table and connect to the proper region servers before starting operations. Defaults to false. Note: This module includes some Google Guava source files from version 12 that were later removed but are still required by HBase's test modules for setting up the mini cluster during integration testing. ================================================ FILE: asynchbase/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ asynchbase-binding AsyncHBase Client Binding for Apache HBase true org.hbase asynchbase ${asynchbase.version} com.yahoo.ycsb core ${project.version} provided org.apache.zookeeper zookeeper 3.4.5 log4j log4j org.slf4j slf4j-log4j12 jline jline junit junit org.jboss.netty netty junit junit 4.12 test org.apache.hbase hbase-testing-util ${hbase10.version} test jdk.tools jdk.tools org.apache.hbase hbase-client ${hbase10.version} test jdk.tools jdk.tools log4j log4j 1.2.17 test org.slf4j log4j-over-slf4j 1.7.7 test org.apache.maven.plugins maven-surefire-plugin 2.20 -Xms4096m -Xms4096m ================================================ FILE: asynchbase/src/main/java/com/yahoo/ycsb/db/AsyncHBaseClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.Vector; import org.hbase.async.Bytes; import org.hbase.async.Config; import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; import org.hbase.async.Scanner; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; /** * Alternative Java client for Apache HBase. * * This client provides a subset of the main HBase client and uses a completely * asynchronous pipeline for all calls. It is particularly useful for write heavy * workloads. It is also compatible with all production versions of HBase. */ public class AsyncHBaseClient extends com.yahoo.ycsb.DB { public static final Charset UTF8_CHARSET = Charset.forName("UTF8"); private static final String CLIENT_SIDE_BUFFERING_PROPERTY = "clientbuffering"; private static final String DURABILITY_PROPERTY = "durability"; private static final String PREFETCH_META_PROPERTY = "prefetchmeta"; private static final String CONFIG_PROPERTY = "config"; private static final String COLUMN_FAMILY_PROPERTY = "columnfamily"; private static final String JOIN_TIMEOUT_PROPERTY = "jointimeout"; private static final String JOIN_TIMEOUT_PROPERTY_DEFAULT = "30000"; /** Mutex for instantiating a single instance of the client. */ private static final Object MUTEX = new Object(); /** Use for tracking running thread counts so we know when to shutdown the client. */ private static int threadCount = 0; /** The client that's used for all threads. */ private static HBaseClient client; /** Print debug information to standard out. */ private boolean debug = false; /** The column family use for the workload. */ private byte[] columnFamilyBytes; /** Cache for the last table name/ID to avoid byte conversions. */ private String lastTable = ""; private byte[] lastTableBytes; private long joinTimeout; /** Whether or not to bypass the WAL for puts and deletes. */ private boolean durability = true; /** * If true, buffer mutations on the client. This is the default behavior for * AsyncHBase. For measuring insert/update/delete latencies, client side * buffering should be disabled. * * A single instance of this */ private boolean clientSideBuffering = false; @Override public void init() throws DBException { if (getProperties().getProperty(CLIENT_SIDE_BUFFERING_PROPERTY, "false") .toLowerCase().equals("true")) { clientSideBuffering = true; } if (getProperties().getProperty(DURABILITY_PROPERTY, "true") .toLowerCase().equals("false")) { durability = false; } final String columnFamily = getProperties().getProperty(COLUMN_FAMILY_PROPERTY); if (columnFamily == null || columnFamily.isEmpty()) { System.err.println("Error, must specify a columnfamily for HBase table"); throw new DBException("No columnfamily specified"); } columnFamilyBytes = columnFamily.getBytes(); if ((getProperties().getProperty("debug") != null) && (getProperties().getProperty("debug").compareTo("true") == 0)) { debug = true; } joinTimeout = Integer.parseInt(getProperties().getProperty( JOIN_TIMEOUT_PROPERTY, JOIN_TIMEOUT_PROPERTY_DEFAULT)); final boolean prefetchMeta = getProperties() .getProperty(PREFETCH_META_PROPERTY, "false") .toLowerCase().equals("true") ? true : false; try { synchronized (MUTEX) { ++threadCount; if (client == null) { final String configPath = getProperties().getProperty(CONFIG_PROPERTY); final Config config; if (configPath == null || configPath.isEmpty()) { config = new Config(); final Iterator> iterator = getProperties() .entrySet().iterator(); while (iterator.hasNext()) { final Entry property = iterator.next(); config.overrideConfig((String)property.getKey(), (String)property.getValue()); } } else { config = new Config(configPath); } client = new HBaseClient(config); // Terminate right now if table does not exist, since the client // will not propagate this error upstream once the workload // starts. String table = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); try { client.ensureTableExists(table).join(joinTimeout); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); } catch (Exception e) { throw new DBException(e); } if (prefetchMeta) { try { if (debug) { System.out.println("Starting meta prefetch for table " + table); } client.prefetchMeta(table).join(joinTimeout); if (debug) { System.out.println("Completed meta prefetch for table " + table); } } catch (InterruptedException e) { System.err.println("Interrupted during prefetch"); Thread.currentThread().interrupt(); } catch (Exception e) { throw new DBException("Failed prefetch", e); } } } } } catch (IOException e) { throw new DBException("Failed instantiation of client", e); } } @Override public void cleanup() throws DBException { synchronized (MUTEX) { --threadCount; if (client != null && threadCount < 1) { try { if (debug) { System.out.println("Shutting down client"); } client.shutdown().joinUninterruptibly(joinTimeout); } catch (Exception e) { System.err.println("Failed to shutdown the AsyncHBase client " + "properly: " + e.getMessage()); } client = null; } } } @Override public Status read(String table, String key, Set fields, Map result) { setTable(table); final GetRequest get = new GetRequest( lastTableBytes, key.getBytes(), columnFamilyBytes); if (fields != null) { get.qualifiers(getQualifierList(fields)); } try { if (debug) { System.out.println("Doing read from HBase columnfamily " + Bytes.pretty(columnFamilyBytes)); System.out.println("Doing read for key: " + key); } final ArrayList row = client.get(get).join(joinTimeout); if (row == null || row.isEmpty()) { return Status.NOT_FOUND; } // got something so populate the results for (final KeyValue column : row) { result.put(new String(column.qualifier()), // TODO - do we need to clone this array? YCSB may keep it in memory // for a while which would mean the entire KV would hang out and won't // be GC'd. new ByteArrayByteIterator(column.value())); if (debug) { System.out.println( "Result for field: " + Bytes.pretty(column.qualifier()) + " is: " + Bytes.pretty(column.value())); } } return Status.OK; } catch (InterruptedException e) { System.err.println("Thread interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { System.err.println("Failure reading from row with key " + key + ": " + e.getMessage()); return Status.ERROR; } return Status.ERROR; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { setTable(table); final Scanner scanner = client.newScanner(lastTableBytes); scanner.setFamily(columnFamilyBytes); scanner.setStartKey(startkey.getBytes(UTF8_CHARSET)); // No end key... *sniff* if (fields != null) { scanner.setQualifiers(getQualifierList(fields)); } // no filters? *sniff* ArrayList> rows = null; try { int numResults = 0; while ((rows = scanner.nextRows().join(joinTimeout)) != null) { for (final ArrayList row : rows) { final HashMap rowResult = new HashMap(row.size()); for (final KeyValue column : row) { rowResult.put(new String(column.qualifier()), // TODO - do we need to clone this array? YCSB may keep it in memory // for a while which would mean the entire KV would hang out and won't // be GC'd. new ByteArrayByteIterator(column.value())); if (debug) { System.out.println("Got scan result for key: " + Bytes.pretty(column.key())); } } result.add(rowResult); numResults++; if (numResults >= recordcount) {// if hit recordcount, bail out break; } } } scanner.close().join(joinTimeout); return Status.OK; } catch (InterruptedException e) { System.err.println("Thread interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { System.err.println("Failure reading from row with key " + startkey + ": " + e.getMessage()); return Status.ERROR; } return Status.ERROR; } @Override public Status update(String table, String key, Map values) { setTable(table); if (debug) { System.out.println("Setting up put for key: " + key); } final byte[][] qualifiers = new byte[values.size()][]; final byte[][] byteValues = new byte[values.size()][]; int idx = 0; for (final Entry entry : values.entrySet()) { qualifiers[idx] = entry.getKey().getBytes(); byteValues[idx++] = entry.getValue().toArray(); if (debug) { System.out.println("Adding field/value " + entry.getKey() + "/" + Bytes.pretty(entry.getValue().toArray()) + " to put request"); } } final PutRequest put = new PutRequest(lastTableBytes, key.getBytes(), columnFamilyBytes, qualifiers, byteValues); if (!durability) { put.setDurable(false); } if (!clientSideBuffering) { put.setBufferable(false); try { client.put(put).join(joinTimeout); } catch (InterruptedException e) { System.err.println("Thread interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { System.err.println("Failure reading from row with key " + key + ": " + e.getMessage()); return Status.ERROR; } } else { // hooray! Asynchronous write. But without a callback and an async // YCSB call we don't know whether it succeeded or not client.put(put); } return Status.OK; } @Override public Status insert(String table, String key, Map values) { return update(table, key, values); } @Override public Status delete(String table, String key) { setTable(table); if (debug) { System.out.println("Doing delete for key: " + key); } final DeleteRequest delete = new DeleteRequest( lastTableBytes, key.getBytes(), columnFamilyBytes); if (!durability) { delete.setDurable(false); } if (!clientSideBuffering) { delete.setBufferable(false); try { client.delete(delete).join(joinTimeout); } catch (InterruptedException e) { System.err.println("Thread interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { System.err.println("Failure reading from row with key " + key + ": " + e.getMessage()); return Status.ERROR; } } else { // hooray! Asynchronous write. But without a callback and an async // YCSB call we don't know whether it succeeded or not client.delete(delete); } return Status.OK; } /** * Little helper to set the table byte array. If it's different than the last * table we reset the byte array. Otherwise we just use the existing array. * @param table The table we're operating against */ private void setTable(final String table) { if (!lastTable.equals(table)) { lastTable = table; lastTableBytes = table.getBytes(); } } /** * Little helper to build a qualifier byte array from a field set. * @param fields The fields to fetch. * @return The column qualifier byte arrays. */ private byte[][] getQualifierList(final Set fields) { final byte[][] qualifiers = new byte[fields.size()][]; int idx = 0; for (final String field : fields) { qualifiers[idx++] = field.getBytes(); } return qualifiers; } } ================================================ FILE: asynchbase/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for HBase using the AsyncHBase client. */ package com.yahoo.ycsb.db; ================================================ FILE: asynchbase/src/test/java/com/google/common/base/Stopwatch.java ================================================ /* * Copyright (C) 2008 The Guava Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.common.base; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import com.google.common.annotations.Beta; import com.google.common.annotations.GwtCompatible; import com.google.common.annotations.GwtIncompatible; import java.util.concurrent.TimeUnit; /** * An object that measures elapsed time in nanoseconds. It is useful to measure * elapsed time using this class instead of direct calls to {@link * System#nanoTime} for a few reasons: * *
    *
  • An alternate time source can be substituted, for testing or performance * reasons. *
  • As documented by {@code nanoTime}, the value returned has no absolute * meaning, and can only be interpreted as relative to another timestamp * returned by {@code nanoTime} at a different time. {@code Stopwatch} is a * more effective abstraction because it exposes only these relative values, * not the absolute ones. *
* *

Basic usage: * 

 *   Stopwatch stopwatch = Stopwatch.{@link #createStarted createStarted}();
 *   doSomething();
 *   stopwatch.{@link #stop stop}(); // optional
 *
 *   long millis = stopwatch.elapsed(MILLISECONDS);
 *
 *   log.info("that took: " + stopwatch); // formatted string like "12.3 ms"
 *
* *

Stopwatch methods are not idempotent; it is an error to start or stop a * stopwatch that is already in the desired state. * *

When testing code that uses this class, use the {@linkplain * #Stopwatch(Ticker) alternate constructor} to supply a fake or mock ticker. * This allows you to * simulate any valid behavior of the stopwatch. * *

Note: This class is not thread-safe. * * @author Kevin Bourrillion * @since 10.0 */ @Beta @GwtCompatible(emulated = true) public final class Stopwatch { private final Ticker ticker; private boolean isRunning; private long elapsedNanos; private long startTick; /** * Creates (but does not start) a new stopwatch using {@link System#nanoTime} * as its time source. * * @since 15.0 */ public static Stopwatch createUnstarted() { return new Stopwatch(); } /** * Creates (but does not start) a new stopwatch, using the specified time * source. * * @since 15.0 */ public static Stopwatch createUnstarted(Ticker ticker) { return new Stopwatch(ticker); } /** * Creates (and starts) a new stopwatch using {@link System#nanoTime} * as its time source. * * @since 15.0 */ public static Stopwatch createStarted() { return new Stopwatch().start(); } /** * Creates (and starts) a new stopwatch, using the specified time * source. * * @since 15.0 */ public static Stopwatch createStarted(Ticker ticker) { return new Stopwatch(ticker).start(); } /** * Creates (but does not start) a new stopwatch using {@link System#nanoTime} * as its time source. * * @deprecated Use {@link Stopwatch#createUnstarted()} instead. */ @Deprecated public Stopwatch() { this(Ticker.systemTicker()); } /** * Creates (but does not start) a new stopwatch, using the specified time * source. * * @deprecated Use {@link Stopwatch#createUnstarted(Ticker)} instead. */ @Deprecated public Stopwatch(Ticker ticker) { this.ticker = checkNotNull(ticker, "ticker"); } /** * Returns {@code true} if {@link #start()} has been called on this stopwatch, * and {@link #stop()} has not been called since the last call to {@code * start()}. */ public boolean isRunning() { return isRunning; } /** * Starts the stopwatch. * * @return this {@code Stopwatch} instance * @throws IllegalStateException if the stopwatch is already running. */ public Stopwatch start() { checkState(!isRunning, "This stopwatch is already running."); isRunning = true; startTick = ticker.read(); return this; } /** * Stops the stopwatch. Future reads will return the fixed duration that had * elapsed up to this point. * * @return this {@code Stopwatch} instance * @throws IllegalStateException if the stopwatch is already stopped. */ public Stopwatch stop() { long tick = ticker.read(); checkState(isRunning, "This stopwatch is already stopped."); isRunning = false; elapsedNanos += tick - startTick; return this; } /** * Sets the elapsed time for this stopwatch to zero, * and places it in a stopped state. * * @return this {@code Stopwatch} instance */ public Stopwatch reset() { elapsedNanos = 0; isRunning = false; return this; } private long elapsedNanos() { return isRunning ? ticker.read() - startTick + elapsedNanos : elapsedNanos; } /** * Returns the current elapsed time shown on this stopwatch, expressed * in the desired time unit, with any fraction rounded down. * *

Note that the overhead of measurement can be more than a microsecond, so * it is generally not useful to specify {@link TimeUnit#NANOSECONDS} * precision here. * * @since 14.0 (since 10.0 as {@code elapsedTime()}) */ public long elapsed(TimeUnit desiredUnit) { return desiredUnit.convert(elapsedNanos(), NANOSECONDS); } /** * Returns the current elapsed time shown on this stopwatch, expressed * in the desired time unit, with any fraction rounded down. * *

Note that the overhead of measurement can be more than a microsecond, so * it is generally not useful to specify {@link TimeUnit#NANOSECONDS} * precision here. * * @deprecated Use {@link Stopwatch#elapsed(TimeUnit)} instead. This method is * scheduled to be removed in Guava release 16.0. */ @Deprecated public long elapsedTime(TimeUnit desiredUnit) { return elapsed(desiredUnit); } /** * Returns the current elapsed time shown on this stopwatch, expressed * in milliseconds, with any fraction rounded down. This is identical to * {@code elapsed(TimeUnit.MILLISECONDS)}. * * @deprecated Use {@code stopwatch.elapsed(MILLISECONDS)} instead. This * method is scheduled to be removed in Guava release 16.0. */ @Deprecated public long elapsedMillis() { return elapsed(MILLISECONDS); } /** * Returns a string representation of the current elapsed time. */ @GwtIncompatible("String.format()") @Override public String toString() { long nanos = elapsedNanos(); TimeUnit unit = chooseUnit(nanos); double value = (double) nanos / NANOSECONDS.convert(1, unit); // Too bad this functionality is not exposed as a regular method call return String.format("%.4g %s", value, abbreviate(unit)); } private static TimeUnit chooseUnit(long nanos) { if (SECONDS.convert(nanos, NANOSECONDS) > 0) { return SECONDS; } if (MILLISECONDS.convert(nanos, NANOSECONDS) > 0) { return MILLISECONDS; } if (MICROSECONDS.convert(nanos, NANOSECONDS) > 0) { return MICROSECONDS; } return NANOSECONDS; } private static String abbreviate(TimeUnit unit) { switch (unit) { case NANOSECONDS: return "ns"; case MICROSECONDS: return "\u03bcs"; // μs case MILLISECONDS: return "ms"; case SECONDS: return "s"; default: throw new AssertionError(); } } } ================================================ FILE: asynchbase/src/test/java/com/google/common/io/Closeables.java ================================================ /* * Copyright (C) 2007 The Guava Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.common.io; import com.google.common.annotations.Beta; import com.google.common.annotations.VisibleForTesting; import java.io.Closeable; import java.io.IOException; import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; /** * Utility methods for working with {@link Closeable} objects. * * @author Michael Lancaster * @since 1.0 */ @Beta public final class Closeables { @VisibleForTesting static final Logger logger = Logger.getLogger(Closeables.class.getName()); private Closeables() {} /** * Closes a {@link Closeable}, with control over whether an * {@code IOException} may be thrown. This is primarily useful in a * finally block, where a thrown exception needs to be logged but not * propagated (otherwise the original exception will be lost). * *

If {@code swallowIOException} is true then we never throw * {@code IOException} but merely log it. * *

Example: * * 

public void useStreamNicely() throws IOException {
   * SomeStream stream = new SomeStream("foo");
   * boolean threw = true;
   * try {
   *   // Some code which does something with the Stream. May throw a
   *   // Throwable.
   *   threw = false; // No throwable thrown.
   * } finally {
   *   // Close the stream.
   *   // If an exception occurs, only rethrow it if (threw==false).
   *   Closeables.close(stream, threw);
   * }
   *
* * @param closeable the {@code Closeable} object to be closed, or null, * in which case this method does nothing * @param swallowIOException if true, don't propagate IO exceptions * thrown by the {@code close} methods * @throws IOException if {@code swallowIOException} is false and * {@code close} throws an {@code IOException}. */ public static void close(@Nullable Closeable closeable, boolean swallowIOException) throws IOException { if (closeable == null) { return; } try { closeable.close(); } catch (IOException e) { if (swallowIOException) { logger.log(Level.WARNING, "IOException thrown while closing Closeable.", e); } else { throw e; } } } /** * Equivalent to calling {@code close(closeable, true)}, but with no * IOException in the signature. * @param closeable the {@code Closeable} object to be closed, or null, in * which case this method does nothing */ public static void closeQuietly(@Nullable Closeable closeable) { try { close(closeable, true); } catch (IOException e) { logger.log(Level.SEVERE, "IOException should not have been thrown.", e); } } } ================================================ FILE: asynchbase/src/test/java/com/google/common/io/LimitInputStream.java ================================================ /* * Copyright (C) 2007 The Guava Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.common.io; import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; /** * An InputStream that limits the number of bytes which can be read. * * @author Charles Fry * @since 1.0 */ @Beta public final class LimitInputStream extends FilterInputStream { private long left; private long mark = -1; /** * Wraps another input stream, limiting the number of bytes which can be read. * * @param in the input stream to be wrapped * @param limit the maximum number of bytes to be read */ public LimitInputStream(InputStream in, long limit) { super(in); Preconditions.checkNotNull(in); Preconditions.checkArgument(limit >= 0, "limit must be non-negative"); left = limit; } @Override public int available() throws IOException { return (int) Math.min(in.available(), left); } @Override public synchronized void mark(int readlimit) { in.mark(readlimit); mark = left; // it's okay to mark even if mark isn't supported, as reset won't work } @Override public int read() throws IOException { if (left == 0) { return -1; } int result = in.read(); if (result != -1) { --left; } return result; } @Override public int read(byte[] b, int off, int len) throws IOException { if (left == 0) { return -1; } len = (int) Math.min(len, left); int result = in.read(b, off, len); if (result != -1) { left -= result; } return result; } @Override public synchronized void reset() throws IOException { if (!in.markSupported()) { throw new IOException("Mark not supported"); } if (mark == -1) { throw new IOException("Mark not set"); } in.reset(); left = mark; } @Override public long skip(long n) throws IOException { n = Math.min(n, left); long skipped = in.skip(n); left -= skipped; return skipped; } } ================================================ FILE: asynchbase/src/test/java/com/yahoo/ycsb/db/AsyncHBaseTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.workloads.CoreWorkload; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Properties; import java.util.Vector; /** * Integration tests for the YCSB AsyncHBase client, using an HBase minicluster. * These are the same as those for the hbase10 client. */ public class AsyncHBaseTest { private final static String COLUMN_FAMILY = "cf"; private static HBaseTestingUtility testingUtil; private AsyncHBaseClient client; private Table table = null; private String tableName; private static boolean isWindows() { final String os = System.getProperty("os.name"); return os.startsWith("Windows"); } /** * Creates a mini-cluster for use in these tests. * * This is a heavy-weight operation, so invoked only once for the test class. */ @BeforeClass public static void setUpClass() throws Exception { // Minicluster setup fails on Windows with an UnsatisfiedLinkError. // Skip if windows. assumeTrue(!isWindows()); testingUtil = HBaseTestingUtility.createLocalHTU(); testingUtil.startMiniCluster(); } /** * Tears down mini-cluster. */ @AfterClass public static void tearDownClass() throws Exception { if (testingUtil != null) { testingUtil.shutdownMiniCluster(); } } /** * Sets up the mini-cluster for testing. * * We re-create the table for each test. */ @Before public void setUp() throws Exception { Properties p = new Properties(); p.setProperty("columnfamily", COLUMN_FAMILY); Measurements.setProperties(p); final CoreWorkload workload = new CoreWorkload(); workload.init(p); tableName = p.getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); table = testingUtil.createTable(TableName.valueOf(tableName), Bytes.toBytes(COLUMN_FAMILY)); final String zkQuorum = "127.0.0.1:" + testingUtil.getZkCluster().getClientPort(); p.setProperty("hbase.zookeeper.quorum", zkQuorum); client = new AsyncHBaseClient(); client.setProperties(p); client.init(); } @After public void tearDown() throws Exception { table.close(); testingUtil.deleteTable(tableName); } @Test public void testRead() throws Exception { final String rowKey = "row1"; final Put p = new Put(Bytes.toBytes(rowKey)); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes("column1"), Bytes.toBytes("value1")); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes("column2"), Bytes.toBytes("value2")); table.put(p); final HashMap result = new HashMap(); final Status status = client.read(tableName, rowKey, null, result); assertEquals(Status.OK, status); assertEquals(2, result.size()); assertEquals("value1", result.get("column1").toString()); assertEquals("value2", result.get("column2").toString()); } @Test public void testReadMissingRow() throws Exception { final HashMap result = new HashMap(); final Status status = client.read(tableName, "Missing row", null, result); assertEquals(Status.NOT_FOUND, status); assertEquals(0, result.size()); } @Test public void testScan() throws Exception { // Fill with data final String colStr = "row_number"; final byte[] col = Bytes.toBytes(colStr); final int n = 10; final List puts = new ArrayList(n); for(int i = 0; i < n; i++) { final byte[] key = Bytes.toBytes(String.format("%05d", i)); final byte[] value = java.nio.ByteBuffer.allocate(4).putInt(i).array(); final Put p = new Put(key); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), col, value); puts.add(p); } table.put(puts); // Test final Vector> result = new Vector>(); // Scan 5 records, skipping the first client.scan(tableName, "00001", 5, null, result); assertEquals(5, result.size()); for(int i = 0; i < 5; i++) { final HashMap row = result.get(i); assertEquals(1, row.size()); assertTrue(row.containsKey(colStr)); final byte[] bytes = row.get(colStr).toArray(); final ByteBuffer buf = ByteBuffer.wrap(bytes); final int rowNum = buf.getInt(); assertEquals(i + 1, rowNum); } } @Test public void testUpdate() throws Exception{ final String key = "key"; final HashMap input = new HashMap(); input.put("column1", "value1"); input.put("column2", "value2"); final Status status = client.insert(tableName, key, StringByteIterator.getByteIteratorMap(input)); assertEquals(Status.OK, status); // Verify result final Get get = new Get(Bytes.toBytes(key)); final Result result = this.table.get(get); assertFalse(result.isEmpty()); assertEquals(2, result.size()); for(final java.util.Map.Entry entry : input.entrySet()) { assertEquals(entry.getValue(), new String(result.getValue(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes(entry.getKey())))); } } @Test @Ignore("Not yet implemented") public void testDelete() { fail("Not yet implemented"); } } ================================================ FILE: asynchbase/src/test/resources/hbase-site.xml ================================================ hbase.master.info.port -1 The port for the hbase master web UI Set to -1 if you do not want the info server to run. hbase.regionserver.info.port -1 The port for the hbase regionserver web UI Set to -1 if you do not want the info server to run. ================================================ FILE: asynchbase/src/test/resources/log4j.properties ================================================ # # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=WARN, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZKTableStateManager: Creates a large number of table # state change messages. log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKTableStateManager=ERROR ================================================ FILE: azuredocumentdb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ azuredocumentdb-binding Azure DocumentDB Binding com.microsoft.azure azure-documentdb ${azuredocumentdb.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: azuredocumentdb/src/main/java/com/yahoo/ycsb/db/azuredocumentdb/AzureDocumentDBClient.java ================================================ /* * Copyright 2016 YCSB Contributors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. See the License * for the specific language governing permissions and limitations under * the License. */ package com.yahoo.ycsb.db.azuredocumentdb; import com.yahoo.ycsb.*; import com.microsoft.azure.documentdb.ConnectionPolicy; import com.microsoft.azure.documentdb.ConsistencyLevel; import com.microsoft.azure.documentdb.Database; import com.microsoft.azure.documentdb.Document; import com.microsoft.azure.documentdb.DocumentClient; import com.microsoft.azure.documentdb.DocumentClientException; import com.microsoft.azure.documentdb.DocumentCollection; import com.microsoft.azure.documentdb.FeedOptions; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.Vector; import java.util.List; /** * Azure DocumentDB client binding. */ public class AzureDocumentDBClient extends DB { private static String host; private static String masterKey; private static String databaseId; private static Database database; private static DocumentClient documentClient; private static DocumentCollection collection; private static FeedOptions feedOptions; @Override public void init() throws DBException { host = getProperties().getProperty("documentdb.host", null); masterKey = getProperties().getProperty("documentdb.masterKey", null); if (host == null) { System.err.println("ERROR: 'documentdb.host' must be set!"); System.exit(1); } if (masterKey == null) { System.err.println("ERROR: 'documentdb.masterKey' must be set!"); System.exit(1); } databaseId = getProperties().getProperty("documentdb.databaseId", "ycsb"); String collectionId = getProperties().getProperty("documentdb.collectionId", "usertable"); documentClient = new DocumentClient(host, masterKey, ConnectionPolicy.GetDefault(), ConsistencyLevel.Session); try { // Initialize test database and collection. collection = getCollection(collectionId); } catch (DocumentClientException e) { throw new DBException("Initialze collection failed", e); } feedOptions = new FeedOptions(); feedOptions.setEmitVerboseTracesInQuery(false); } @Override public Status read(String table, String key, Set fields, Map result) { Document record = getDocumentById(table, key); if (record != null) { Set fieldsToReturn = (fields == null ? record.getHashMap().keySet() : fields); for (String field : fieldsToReturn) { if (field.startsWith("_")) { continue; } result.put(field, new StringByteIterator(record.getString(field))); } return Status.OK; } // Unable to find the specidifed document. return Status.ERROR; } @Override public Status update(String table, String key, Map values) { Document record = getDocumentById(table, key); if (record == null) { return Status.ERROR; } // Update each field. for (Entry val : values.entrySet()) { record.set(val.getKey(), val.getValue().toString()); } // Replace the document. try { documentClient.replaceDocument(record, null); } catch (DocumentClientException e) { e.printStackTrace(System.err); return Status.ERROR; } return Status.OK; } @Override public Status insert(String table, String key, Map values) { Document record = new Document(); record.set("id", key); for (Entry val : values.entrySet()) { record.set(val.getKey(), val.getValue().toString()); } try { documentClient.createDocument(collection.getSelfLink(), record, null, false); } catch (DocumentClientException e) { e.printStackTrace(System.err); return Status.ERROR; } return Status.OK; } @Override public Status delete(String table, String key) { Document record = getDocumentById(table, key); try { // Delete the document by self link. documentClient.deleteDocument(record.getSelfLink(), null); } catch (DocumentClientException e) { e.printStackTrace(); return Status.ERROR; } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // TODO: Implement Scan as query on primary key. return Status.NOT_IMPLEMENTED; } private Database getDatabase() { if (database == null) { // Get the database if it exists List databaseList = documentClient .queryDatabases( "SELECT * FROM root r WHERE r.id='" + databaseId + "'", null) .getQueryIterable() .toList(); if (databaseList.size() > 0) { // Cache the database object so we won't have to query for it // later to retrieve the selfLink. database = databaseList.get(0); } else { // Create the database if it doesn't exist. try { Database databaseDefinition = new Database(); databaseDefinition.setId(databaseId); database = documentClient.createDatabase(databaseDefinition, null) .getResource(); } catch (DocumentClientException e) { // TODO: Something has gone terribly wrong - the app wasn't // able to query or create the collection. // Verify your connection, endpoint, and key. e.printStackTrace(System.err); } } } return database; } private DocumentCollection getCollection(String collectionId) throws DocumentClientException { if (collection == null) { // Get the collection if it exists. List collectionList = documentClient .queryCollections(getDatabase().getSelfLink(), "SELECT * FROM root r WHERE r.id='" + collectionId + "'", null) .getQueryIterable() .toList(); if (collectionList.size() > 0) { // Cache the collection object so we won't have to query for it // later to retrieve the selfLink. collection = collectionList.get(0); } else { // Create the collection if it doesn't exist. try { DocumentCollection collectionDefinition = new DocumentCollection(); collectionDefinition.setId(collectionId); collection = documentClient .createCollection(getDatabase().getSelfLink(), collectionDefinition, null) .getResource(); } catch (DocumentClientException e) { // TODO: Something has gone terribly wrong - the app wasn't // able to query or create the collection. // Verify your connection, endpoint, and key. e.printStackTrace(System.err); throw e; } } } return collection; } private Document getDocumentById(String collectionId, String id) { if (collection == null) { return null; } // Retrieve the document using the DocumentClient. List documentList = documentClient .queryDocuments(collection.getSelfLink(), "SELECT * FROM root r WHERE r.id='" + id + "'", feedOptions) .getQueryIterable() .toList(); if (documentList.size() > 0) { return documentList.get(0); } return null; } } ================================================ FILE: azuredocumentdb/src/main/java/com/yahoo/ycsb/db/azuredocumentdb/package-info.java ================================================ /* * Copyright 2016 YCSB Contributors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Azure DocumentDB. */ package com.yahoo.ycsb.db.azuredocumentdb; ================================================ FILE: azuretablestorage/README.md ================================================ ## Quick Start This section describes how to run YCSB on Azure table storage. ### 1. Create an Azure Storage account. ### https://azure.microsoft.com/en-us/documentation/articles/storage-create-storage-account/#create-a-storage-account ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:azuretablestorage-binding -am clean package ### 4. Provide Azure Storage parameters Set the account name and access key. - `azure.account` - `azure.key` Or, you can set configs with the shell command, EG: ./bin/ycsb load azuretablestorage -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey > outputLoad.txt ### 5. Load data and run tests Load the data: ./bin/ycsb load azuretablestorage -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey > outputLoad.txt Run the workload test: ./bin/ycsb run azuretablestorage -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey > outputRun.txt ### 6. Optional Azure Storage parameters - `azure.batchsize` Could be between 1 ~ 100. Insert records to table in batch if batchsize > 1. - `azure.protocol` https(in default) or http. - `azure.table` The name of the table('usertable' in default). - `azure.partitionkey` The partitionkey('Test' in default). - `azure.endpoint` For Azure stack WOSS. EG: ./bin/ycsb load azuretablestorage -s -P workloads/workloada -p azure.account=YourAccountName -p azure.key=YourAccessKey -p azure.batchsize=100 -p azure.protocol=http ================================================ FILE: azuretablestorage/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent azuretablestorage-binding Azure table storage Binding jar com.yahoo.ycsb core ${project.version} provided com.microsoft.azure azure-storage ${azurestorage.version} ================================================ FILE: azuretablestorage/src/main/java/com/yahoo/ycsb/db/azuretablestorage/AzureClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.azuretablestorage; import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.table.CloudTable; import com.microsoft.azure.storage.table.CloudTableClient; import com.microsoft.azure.storage.table.DynamicTableEntity; import com.microsoft.azure.storage.table.EntityProperty; import com.microsoft.azure.storage.table.EntityResolver; import com.microsoft.azure.storage.table.TableBatchOperation; import com.microsoft.azure.storage.table.TableOperation; import com.microsoft.azure.storage.table.TableQuery; import com.microsoft.azure.storage.table.TableServiceEntity; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * YCSB binding for Azure. * See {@code azure/README.md} for details. */ public class AzureClient extends DB { public static final String PROTOCOL = "azure.protocal"; public static final String PROTOCOL_DEFAULT = "https"; public static final String TABLE_ENDPOINT = "azure.endpoint"; public static final String ACCOUNT = "azure.account"; public static final String KEY = "azure.key"; public static final String TABLE = "azure.table"; public static final String TABLE_DEFAULT = "usertable"; public static final String PARTITIONKEY = "azure.partitionkey"; public static final String PARTITIONKEY_DEFAULT = "Test"; public static final String BATCHSIZE = "azure.batchsize"; public static final String BATCHSIZE_DEFAULT = "1"; private static final int BATCHSIZE_UPPERBOUND = 100; private static final TableBatchOperation BATCH_OPERATION = new TableBatchOperation(); private static String partitionKey; private CloudStorageAccount storageAccount = null; private CloudTableClient tableClient = null; private CloudTable cloudTable = null; private static int batchSize; private static int curIdx = 0; @Override public void init() throws DBException { Properties props = getProperties(); String protocol = props.getProperty(PROTOCOL, PROTOCOL_DEFAULT); if (protocol != "https" && protocol != "http") { throw new DBException("Protocol must be 'http' or 'https'!\n"); } String table = props.getProperty(TABLE, TABLE_DEFAULT); partitionKey = props.getProperty(PARTITIONKEY, PARTITIONKEY_DEFAULT); batchSize = Integer.parseInt(props.getProperty(BATCHSIZE, BATCHSIZE_DEFAULT)); if (batchSize < 1 || batchSize > BATCHSIZE_UPPERBOUND) { throw new DBException(String.format("Batchsize must be between 1 and %d!\n", BATCHSIZE_UPPERBOUND)); } String account = props.getProperty(ACCOUNT); String key = props.getProperty(KEY); String tableEndPoint = props.getProperty(TABLE_ENDPOINT); String storageConnectionString = getStorageConnectionString(protocol, account, key, tableEndPoint); try { storageAccount = CloudStorageAccount.parse(storageConnectionString); } catch (Exception e) { throw new DBException("Could not connect to the account.\n", e); } tableClient = storageAccount.createCloudTableClient(); try { cloudTable = tableClient.getTableReference(table); cloudTable.createIfNotExists(); } catch (Exception e) { throw new DBException("Could not connect to the table.\n", e); } } @Override public void cleanup() { } @Override public Status read(String table, String key, Set fields, Map result) { if (fields != null) { return readSubset(key, fields, result); } else { return readEntity(key, result); } } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { String whereStr = String.format("(PartitionKey eq '%s') and (RowKey ge '%s')", partitionKey, startkey); TableQuery scanQuery = new TableQuery(DynamicTableEntity.class) .where(whereStr).take(recordcount); int cnt = 0; for (DynamicTableEntity entity : cloudTable.execute(scanQuery)) { HashMap properties = entity.getProperties(); HashMap cur = new HashMap(); for (Entry entry : properties.entrySet()) { String fieldName = entry.getKey(); ByteIterator fieldVal = new ByteArrayByteIterator(entry.getValue().getValueAsByteArray()); if (fields == null || fields.contains(fieldName)) { cur.put(fieldName, fieldVal); } } result.add(cur); if (++cnt == recordcount) { break; } } return Status.OK; } catch (Exception e) { return Status.ERROR; } } @Override public Status update(String table, String key, Map values) { return insertOrUpdate(key, values); } @Override public Status insert(String table, String key, Map values) { if (batchSize == 1) { return insertOrUpdate(key, values); } else { return insertBatch(key, values); } } @Override public Status delete(String table, String key) { try { // firstly, retrieve the entity to be deleted TableOperation retrieveOp = TableOperation.retrieve(partitionKey, key, TableServiceEntity.class); TableServiceEntity entity = cloudTable.execute(retrieveOp).getResultAsType(); // secondly, delete the entity TableOperation deleteOp = TableOperation.delete(entity); cloudTable.execute(deleteOp); return Status.OK; } catch (Exception e) { return Status.ERROR; } } private String getStorageConnectionString(String protocol, String account, String key, String tableEndPoint) { String res = String.format("DefaultEndpointsProtocol=%s;AccountName=%s;AccountKey=%s", protocol, account, key); if (tableEndPoint != null) { res = String.format("%s;TableEndpoint=%s", res, tableEndPoint); } return res; } /* * Read subset of properties instead of full fields with projection. */ public Status readSubset(String key, Set fields, Map result) { String whereStr = String.format("RowKey eq '%s'", key); TableQuery projectionQuery = TableQuery.from( TableServiceEntity.class).where(whereStr).select(fields.toArray(new String[0])); EntityResolver> resolver = new EntityResolver>() { public HashMap resolve(String partitionkey, String rowKey, Date timeStamp, HashMap properties, String etag) { HashMap tmp = new HashMap(); for (Entry entry : properties.entrySet()) { String key = entry.getKey(); ByteIterator val = new ByteArrayByteIterator(entry.getValue().getValueAsByteArray()); tmp.put(key, val); } return tmp; } }; try { for (HashMap tmp : cloudTable.execute(projectionQuery, resolver)) { for (Entry entry : tmp.entrySet()){ String fieldName = entry.getKey(); ByteIterator fieldVal = entry.getValue(); result.put(fieldName, fieldVal); } } return Status.OK; } catch (Exception e) { return Status.ERROR; } } private Status readEntity(String key, Map result) { try { // firstly, retrieve the entity to be deleted TableOperation retrieveOp = TableOperation.retrieve(partitionKey, key, DynamicTableEntity.class); DynamicTableEntity entity = cloudTable.execute(retrieveOp).getResultAsType(); HashMap properties = entity.getProperties(); for (Entry entry: properties.entrySet()) { String fieldName = entry.getKey(); ByteIterator fieldVal = new ByteArrayByteIterator(entry.getValue().getValueAsByteArray()); result.put(fieldName, fieldVal); } return Status.OK; } catch (Exception e) { return Status.ERROR; } } private Status insertBatch(String key, Map values) { HashMap properties = new HashMap(); for (Entry entry : values.entrySet()) { String fieldName = entry.getKey(); byte[] fieldVal = entry.getValue().toArray(); properties.put(fieldName, new EntityProperty(fieldVal)); } DynamicTableEntity entity = new DynamicTableEntity(partitionKey, key, properties); BATCH_OPERATION.insertOrReplace(entity); if (++curIdx == batchSize) { try { cloudTable.execute(BATCH_OPERATION); BATCH_OPERATION.clear(); curIdx = 0; } catch (Exception e) { return Status.ERROR; } } return Status.OK; } private Status insertOrUpdate(String key, Map values) { HashMap properties = new HashMap(); for (Entry entry : values.entrySet()) { String fieldName = entry.getKey(); byte[] fieldVal = entry.getValue().toArray(); properties.put(fieldName, new EntityProperty(fieldVal)); } DynamicTableEntity entity = new DynamicTableEntity(partitionKey, key, properties); TableOperation insertOrReplace = TableOperation.insertOrReplace(entity); try { cloudTable.execute(insertOrReplace); return Status.OK; } catch (Exception e) { return Status.ERROR; } } } ================================================ FILE: azuretablestorage/src/main/java/com/yahoo/ycsb/db/azuretablestorage/package-info.java ================================================ /* * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Azure table Storage. */ package com.yahoo.ycsb.db.azuretablestorage; ================================================ FILE: bin/bindings.properties ================================================ # # Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # #DATABASE BINDINGS # # Available bindings should be listed here in the form of # name:class # # - the name must start in column 0. # - the name is also the directory where the class can be found. # - if the directory contains multiple versions with different classes, # use a dash with the version. (e.g. cassandra-7, cassandra-cql) # accumulo:com.yahoo.ycsb.db.accumulo.AccumuloClient accumulo1.6:com.yahoo.ycsb.db.accumulo.AccumuloClient accumulo1.7:com.yahoo.ycsb.db.accumulo.AccumuloClient accumulo1.8:com.yahoo.ycsb.db.accumulo.AccumuloClient aerospike:com.yahoo.ycsb.db.AerospikeClient asynchbase:com.yahoo.ycsb.db.AsyncHBaseClient arangodb:com.yahoo.ycsb.db.ArangoDBClient arangodb3:com.yahoo.ycsb.db.arangodb.ArangoDB3Client azuredocumentdb:com.yahoo.ycsb.db.azuredocumentdb.AzureDocumentDBClient azuretablestorage:com.yahoo.ycsb.db.azuretablestorage.AzureClient basic:com.yahoo.ycsb.BasicDB basicts:com.yahoo.ycsb.BasicTSDB cassandra-cql:com.yahoo.ycsb.db.CassandraCQLClient cassandra2-cql:com.yahoo.ycsb.db.CassandraCQLClient cloudspanner:com.yahoo.ycsb.db.cloudspanner.CloudSpannerClient couchbase:com.yahoo.ycsb.db.CouchbaseClient couchbase2:com.yahoo.ycsb.db.couchbase2.Couchbase2Client dynamodb:com.yahoo.ycsb.db.DynamoDBClient elasticsearch:com.yahoo.ycsb.db.ElasticsearchClient elasticsearch5:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient elasticsearch5-rest:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchRestClient geode:com.yahoo.ycsb.db.GeodeClient googlebigtable:com.yahoo.ycsb.db.GoogleBigtableClient googledatastore:com.yahoo.ycsb.db.GoogleDatastoreClient hbase098:com.yahoo.ycsb.db.HBaseClient hbase10:com.yahoo.ycsb.db.HBaseClient10 hbase12:com.yahoo.ycsb.db.hbase12.HBaseClient12 hypertable:com.yahoo.ycsb.db.HypertableClient infinispan-cs:com.yahoo.ycsb.db.InfinispanRemoteClient infinispan:com.yahoo.ycsb.db.InfinispanClient jdbc:com.yahoo.ycsb.db.JdbcDBClient kudu:com.yahoo.ycsb.db.KuduYCSBClient memcached:com.yahoo.ycsb.db.MemcachedClient mongodb:com.yahoo.ycsb.db.MongoDbClient mongodb-async:com.yahoo.ycsb.db.AsyncMongoDbClient nosqldb:com.yahoo.ycsb.db.NoSqlDbClient orientdb:com.yahoo.ycsb.db.OrientDBClient rados:com.yahoo.ycsb.db.RadosClient redis:com.yahoo.ycsb.db.RedisClient rest:com.yahoo.ycsb.webservice.rest.RestClient riak:com.yahoo.ycsb.db.riak.RiakKVClient s3:com.yahoo.ycsb.db.S3Client solr:com.yahoo.ycsb.db.solr.SolrClient solr6:com.yahoo.ycsb.db.solr6.SolrClient tarantool:com.yahoo.ycsb.db.TarantoolClient ================================================ FILE: bin/ycsb ================================================ #!/usr/bin/env python # # Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # import errno import fnmatch import io import os import shlex import sys import subprocess try: mod = __import__('argparse') import argparse except ImportError: print >> sys.stderr, '[ERROR] argparse not found. Try installing it via "pip".' exit(1) BASE_URL = "https://github.com/brianfrankcooper/YCSB/tree/master/" COMMANDS = { "shell" : { "command" : "", "description" : "Interactive mode", "main" : "com.yahoo.ycsb.CommandLine", }, "load" : { "command" : "-load", "description" : "Execute the load phase", "main" : "com.yahoo.ycsb.Client", }, "run" : { "command" : "-t", "description" : "Execute the transaction phase", "main" : "com.yahoo.ycsb.Client", }, } DATABASES = { "accumulo" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "accumulo1.6" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "accumulo1.7" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "accumulo1.8" : "com.yahoo.ycsb.db.accumulo.AccumuloClient", "aerospike" : "com.yahoo.ycsb.db.AerospikeClient", "arangodb" : "com.yahoo.ycsb.db.ArangoDBClient", "arangodb3" : "com.yahoo.ycsb.db.arangodb.ArangoDB3Client", "asynchbase" : "com.yahoo.ycsb.db.AsyncHBaseClient", "azuredocumentdb" : "com.yahoo.ycsb.db.azuredocumentdb.AzureDocumentDBClient", "azuretablestorage" : "com.yahoo.ycsb.db.azuretablestorage.AzureClient", "basic" : "com.yahoo.ycsb.BasicDB", "basicts" : "com.yahoo.ycsb.BasicTSDB", "cassandra-cql": "com.yahoo.ycsb.db.CassandraCQLClient", "cassandra2-cql": "com.yahoo.ycsb.db.CassandraCQLClient", "cloudspanner" : "com.yahoo.ycsb.db.cloudspanner.CloudSpannerClient", "couchbase" : "com.yahoo.ycsb.db.CouchbaseClient", "couchbase2" : "com.yahoo.ycsb.db.couchbase2.Couchbase2Client", "dynamodb" : "com.yahoo.ycsb.db.DynamoDBClient", "elasticsearch": "com.yahoo.ycsb.db.ElasticsearchClient", "elasticsearch5": "com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient", "geode" : "com.yahoo.ycsb.db.GeodeClient", "googlebigtable" : "com.yahoo.ycsb.db.GoogleBigtableClient", "googledatastore" : "com.yahoo.ycsb.db.GoogleDatastoreClient", "hbase098" : "com.yahoo.ycsb.db.HBaseClient", "hbase10" : "com.yahoo.ycsb.db.HBaseClient10", "hbase12" : "com.yahoo.ycsb.db.hbase12.HBaseClient12", "hypertable" : "com.yahoo.ycsb.db.HypertableClient", "infinispan-cs": "com.yahoo.ycsb.db.InfinispanRemoteClient", "infinispan" : "com.yahoo.ycsb.db.InfinispanClient", "jdbc" : "com.yahoo.ycsb.db.JdbcDBClient", "kudu" : "com.yahoo.ycsb.db.KuduYCSBClient", "memcached" : "com.yahoo.ycsb.db.MemcachedClient", "mongodb" : "com.yahoo.ycsb.db.MongoDbClient", "mongodb-async": "com.yahoo.ycsb.db.AsyncMongoDbClient", "nosqldb" : "com.yahoo.ycsb.db.NoSqlDbClient", "orientdb" : "com.yahoo.ycsb.db.OrientDBClient", "rados" : "com.yahoo.ycsb.db.RadosClient", "redis" : "com.yahoo.ycsb.db.RedisClient", "rest" : "com.yahoo.ycsb.webservice.rest.RestClient", "riak" : "com.yahoo.ycsb.db.riak.RiakKVClient", "s3" : "com.yahoo.ycsb.db.S3Client", "solr" : "com.yahoo.ycsb.db.solr.SolrClient", "solr6" : "com.yahoo.ycsb.db.solr6.SolrClient", "tarantool" : "com.yahoo.ycsb.db.TarantoolClient", } OPTIONS = { "-P file" : "Specify workload file", "-p key=value" : "Override workload property", "-s" : "Print status to stderr", "-target n" : "Target ops/sec (default: unthrottled)", "-threads n" : "Number of client threads (default: 1)", "-cp path" : "Additional Java classpath entries", "-jvm-args args" : "Additional arguments to the JVM", } def usage(): output = io.BytesIO() print >> output, "%s command database [options]" % sys.argv[0] print >> output, "\nCommands:" for command in sorted(COMMANDS.keys()): print >> output, " %s %s" % (command.ljust(14), COMMANDS[command]["description"]) print >> output, "\nDatabases:" for db in sorted(DATABASES.keys()): print >> output, " %s %s" % (db.ljust(14), BASE_URL + db.split("-")[0]) print >> output, "\nOptions:" for option in sorted(OPTIONS.keys()): print >> output, " %s %s" % (option.ljust(14), OPTIONS[option]) print >> output, """\nWorkload Files: There are various predefined workloads under workloads/ directory. See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for the list of workload properties.""" return output.getvalue() # Python 2.6 doesn't have check_output. Add the method as it is in Python 2.7 # Based on https://github.com/python/cpython/blob/2.7/Lib/subprocess.py#L545 def check_output(*popenargs, **kwargs): r"""Run command with arguments and return its output as a byte string. If the exit code was non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute and output in the output attribute. The arguments are the same as for the Popen constructor. Example: >>> check_output(["ls", "-l", "/dev/null"]) 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' The stdout argument is not allowed as it is used internally. To capture standard error in the result, use stderr=STDOUT. >>> check_output(["/bin/sh", "-c", ... "ls -l non_existent_file ; exit 0"], ... stderr=STDOUT) 'ls: non_existent_file: No such file or directory\n' """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) output, unused_err = process.communicate() retcode = process.poll() if retcode: cmd = kwargs.get("args") if cmd is None: cmd = popenargs[0] error = subprocess.CalledProcessError(retcode, cmd) error.output = output raise error return output def debug(message): print >> sys.stderr, "[DEBUG] ", message def warn(message): print >> sys.stderr, "[WARN] ", message def error(message): print >> sys.stderr, "[ERROR] ", message def find_jars(dir, glob='*.jar'): jars = [] for (dirpath, dirnames, filenames) in os.walk(dir): for filename in fnmatch.filter(filenames, glob): jars.append(os.path.join(dirpath, filename)) return jars def get_ycsb_home(): dir = os.path.abspath(os.path.dirname(sys.argv[0])) while "LICENSE.txt" not in os.listdir(dir): dir = os.path.join(dir, os.path.pardir) return os.path.abspath(dir) def is_distribution(): # If there's a top level pom, we're a source checkout. otherwise a dist artifact return "pom.xml" not in os.listdir(get_ycsb_home()) # Run the maven dependency plugin to get the local jar paths. # presumes maven can run, so should only be run on source checkouts # will invoke the 'package' goal for the given binding in order to resolve intra-project deps # presumes maven properly handles system-specific path separators # Given module is full module name eg. 'core' or 'couchbase-binding' def get_classpath_from_maven(module): try: debug("Running 'mvn -pl com.yahoo.ycsb:" + module + " -am package -DskipTests " "dependency:build-classpath -DincludeScope=compile -Dmdep.outputFilterFile=true'") mvn_output = check_output(["mvn", "-pl", "com.yahoo.ycsb:" + module, "-am", "package", "-DskipTests", "dependency:build-classpath", "-DincludeScope=compile", "-Dmdep.outputFilterFile=true"]) # the above outputs a "classpath=/path/tojar:/path/to/other/jar" for each module # the last module will be the datastore binding line = [x for x in mvn_output.splitlines() if x.startswith("classpath=")][-1:] return line[0][len("classpath="):] except subprocess.CalledProcessError, err: error("Attempting to generate a classpath from Maven failed " "with return code '" + str(err.returncode) + "'. The output from " "Maven follows, try running " "'mvn -DskipTests package dependency:build=classpath' on your " "own and correct errors." + os.linesep + os.linesep + "mvn output:" + os.linesep + err.output) sys.exit(err.returncode) def main(): p = argparse.ArgumentParser( usage=usage(), formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('-cp', dest='classpath', help="""Additional classpath entries, e.g. '-cp /tmp/hbase-1.0.1.1/conf'. Will be prepended to the YCSB classpath.""") p.add_argument("-jvm-args", default=[], type=shlex.split, help="""Additional arguments to pass to 'java', e.g. '-Xmx4g'""") p.add_argument("command", choices=sorted(COMMANDS), help="""Command to run.""") p.add_argument("database", choices=sorted(DATABASES), help="""Database to test.""") args, remaining = p.parse_known_args() ycsb_home = get_ycsb_home() # Use JAVA_HOME to find java binary if set, otherwise just use PATH. java = "java" java_home = os.getenv("JAVA_HOME") if java_home: java = os.path.join(java_home, "bin", "java") db_classname = DATABASES[args.database] command = COMMANDS[args.command]["command"] main_classname = COMMANDS[args.command]["main"] # Classpath set up binding = args.database.split("-")[0] if binding == "accumulo": warn("The 'accumulo' client has been deprecated in favor of version " "specific bindings. This name still maps to the binding for " "Accumulo 1.6, which is named 'accumulo-1.6'. This alias will " "be removed in a future YCSB release.") binding = "accumulo1.6" if binding == "cassandra2": warn("The 'cassandra2-cql' client has been deprecated. It has been " "renamed to simply 'cassandra-cql'. This alias will be removed" " in the next YCSB release.") binding = "cassandra" if binding == "couchbase": warn("The 'couchbase' client has been deprecated. If you are using " "Couchbase 4.0+ try using the 'couchbase2' client instead.") if is_distribution(): db_dir = os.path.join(ycsb_home, binding + "-binding") # include top-level conf for when we're a binding-specific artifact. # If we add top-level conf to the general artifact, starting here # will allow binding-specific conf to override (because it's prepended) cp = [os.path.join(ycsb_home, "conf")] cp.extend(find_jars(os.path.join(ycsb_home, "lib"))) cp.extend(find_jars(os.path.join(db_dir, "lib"))) else: warn("Running against a source checkout. In order to get our runtime " "dependencies we'll have to invoke Maven. Depending on the state " "of your system, this may take ~30-45 seconds") db_location = "core" if (binding == "basic" or binding == "basicts") else binding project = "core" if (binding == "basic" or binding == "basicts") else binding + "-binding" db_dir = os.path.join(ycsb_home, db_location) # goes first so we can rely on side-effect of package maven_says = get_classpath_from_maven(project) # TODO when we have a version property, skip the glob cp = find_jars(os.path.join(db_dir, "target"), project + "*.jar") # alredy in jar:jar:jar form cp.append(maven_says) cp.insert(0, os.path.join(db_dir, "conf")) classpath = os.pathsep.join(cp) if args.classpath: classpath = os.pathsep.join([args.classpath, classpath]) ycsb_command = ([java] + args.jvm_args + ["-cp", classpath, main_classname, "-db", db_classname] + remaining) if command: ycsb_command.append(command) print >> sys.stderr, " ".join(ycsb_command) try: return subprocess.call(ycsb_command) except OSError as e: if e.errno == errno.ENOENT: error('Command failed. Is java installed and on your PATH?') return 1 else: raise if __name__ == '__main__': sys.exit(main()) ================================================ FILE: bin/ycsb.bat ================================================ @REM @REM Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. @REM @REM Licensed under the Apache License, Version 2.0 (the "License"); you @REM may not use this file except in compliance with the License. You @REM may obtain a copy of the License at @REM @REM http://www.apache.org/licenses/LICENSE-2.0 @REM @REM Unless required by applicable law or agreed to in writing, software @REM distributed under the License is distributed on an "AS IS" BASIS, @REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or @REM implied. See the License for the specific language governing @REM permissions and limitations under the License. See accompanying @REM LICENSE file. @REM @REM ----------------------------------------------------------------------- @REM Control Script for YCSB @REM @REM Environment Variable Prerequisites @REM @REM Do not set the variables in this script. Instead put them into a script @REM setenv.sh in YCSB_HOME/bin to keep your customizations separate. @REM @REM YCSB_HOME (Optional) YCSB installation directory. If not set @REM this script will use the parent directory of where this @REM script is run from. @REM @REM JAVA_HOME (Required) Must point at your Java Development Kit @REM or Java Runtime Environment installation. @REM @REM JAVA_OPTS (Optional) Java runtime options used when any command @REM is executed. @REM @REM WARNING!!! YCSB home must be located in a directory path that doesn't @REM contain spaces. @REM @ECHO OFF SETLOCAL ENABLEDELAYEDEXPANSION @REM Only set YCSB_HOME if not already set PUSHD %~dp0.. IF NOT DEFINED YCSB_HOME SET YCSB_HOME=%CD% POPD @REM Ensure that any extra CLASSPATH variables are set via setenv.bat SET CLASSPATH= @REM Pull in customization options if exist "%YCSB_HOME%\bin\setenv.bat" call "%YCSB_HOME%\bin\setenv.bat" @REM Check if we have a usable JDK IF "%JAVA_HOME%." == "." GOTO noJavaHome IF NOT EXIST "%JAVA_HOME%\bin\java.exe" GOTO noJavaHome GOTO okJava :noJavaHome ECHO The JAVA_HOME environment variable is not defined correctly. GOTO exit :okJava @REM Determine YCSB command argument IF NOT "load" == "%1" GOTO noload SET YCSB_COMMAND=-load SET YCSB_CLASS=com.yahoo.ycsb.Client GOTO gotCommand :noload IF NOT "run" == "%1" GOTO noRun SET YCSB_COMMAND=-t SET YCSB_CLASS=com.yahoo.ycsb.Client GOTO gotCommand :noRun IF NOT "shell" == "%1" GOTO noShell SET YCSB_COMMAND= SET YCSB_CLASS=com.yahoo.ycsb.CommandLine GOTO gotCommand :noShell ECHO [ERROR] Found unknown command '%1' ECHO [ERROR] Expected one of 'load', 'run', or 'shell'. Exiting. GOTO exit :gotCommand @REM Find binding information FOR /F "delims=" %%G in ( 'FINDSTR /B "%2:" %YCSB_HOME%\bin\bindings.properties' ) DO SET "BINDING_LINE=%%G" IF NOT "%BINDING_LINE%." == "." GOTO gotBindingLine ECHO [ERROR] The specified binding '%2' was not found. Exiting. GOTO exit :gotBindingLine @REM Pull out binding name and class FOR /F "tokens=1-2 delims=:" %%G IN ("%BINDING_LINE%") DO ( SET BINDING_NAME=%%G SET BINDING_CLASS=%%H ) @REM Some bindings have multiple versions that are managed in the same @REM directory. @REM They are noted with a '-' after the binding name. @REM (e.g. cassandra-7 & cassandra-8) FOR /F "tokens=1 delims=-" %%G IN ("%BINDING_NAME%") DO ( SET BINDING_DIR=%%G ) @REM The 'basic' binding is core functionality IF NOT "%BINDING_NAME%" == "basic" GOTO noBasic SET BINDING_DIR=core :noBasic @REM Add Top level conf to classpath IF "%CLASSPATH%." == "." GOTO emptyClasspath SET CLASSPATH=%CLASSPATH%;%YCSB_HOME%\conf GOTO confAdded :emptyClasspath SET CLASSPATH=%YCSB_HOME%\conf :confAdded @REM Accumulo deprecation message IF NOT "%BINDING_DIR%" == "accumulo" GOTO notAliasAccumulo echo [WARN] The 'accumulo' client has been deprecated in favor of version specific bindings. This name still maps to the binding for Accumulo 1.6, which is named 'accumulo-1.6'. This alias will be removed in a future YCSB release. SET BINDING_DIR=accumulo1.6 :notAliasAccumulo @REM Cassandra2 deprecation message IF NOT "%BINDING_DIR%" == "cassandra2" GOTO notAliasCassandra echo [WARN] The 'cassandra2-cql' client has been deprecated. It has been renamed to simply 'cassandra-cql'. This alias will be removed in the next YCSB release. SET BINDING_DIR=cassandra :notAliasCassandra @REM Build classpath according to source checkout or release distribution IF EXIST "%YCSB_HOME%\pom.xml" GOTO gotSource @REM Core libraries FOR %%F IN (%YCSB_HOME%\lib\*.jar) DO ( SET CLASSPATH=!CLASSPATH!;%%F% ) @REM Database conf dir IF NOT EXIST "%YCSB_HOME%\%BINDING_DIR%-binding\conf" GOTO noBindingConf set CLASSPATH=%CLASSPATH%;%YCSB_HOME%\%BINDING_DIR%-binding\conf :noBindingConf @REM Database libraries FOR %%F IN (%YCSB_HOME%\%BINDING_DIR%-binding\lib\*.jar) DO ( SET CLASSPATH=!CLASSPATH!;%%F% ) GOTO classpathComplete :gotSource @REM Check for some basic libraries to see if the source has been built. IF EXIST "%YCSB_HOME%\%BINDING_DIR%\target\*.jar" GOTO gotJars @REM Call mvn to build source checkout. IF "%BINDING_NAME%" == "basic" GOTO buildCore SET MVN_PROJECT=%BINDING_DIR%-binding goto gotMvnProject :buildCore SET MVN_PROJECT=core :gotMvnProject ECHO [WARN] YCSB libraries not found. Attempting to build... CALL mvn -pl com.yahoo.ycsb:%MVN_PROJECT% -am package -DskipTests IF %ERRORLEVEL% NEQ 0 ( ECHO [ERROR] Error trying to build project. Exiting. GOTO exit ) :gotJars @REM Core libraries FOR %%F IN (%YCSB_HOME%\core\target\*.jar) DO ( SET CLASSPATH=!CLASSPATH!;%%F% ) @REM Database conf (need to find because location is not consistent) FOR /D /R %YCSB_HOME%\%BINDING_DIR% %%F IN (*) DO ( IF "%%~nxF" == "conf" SET CLASSPATH=!CLASSPATH!;%%F% ) @REM Database libraries FOR %%F IN (%YCSB_HOME%\%BINDING_DIR%\target\*.jar) DO ( SET CLASSPATH=!CLASSPATH!;%%F% ) @REM Database dependency libraries FOR %%F IN (%YCSB_HOME%\%BINDING_DIR%\target\dependency\*.jar) DO ( SET CLASSPATH=!CLASSPATH!;%%F% ) :classpathComplete @REM Couchbase deprecation message IF NOT "%BINDING_DIR%" == "couchbase" GOTO notOldCouchbase echo [WARN] The 'couchbase' client is deprecated. If you are using Couchbase 4.0+ try using the 'couchbase2' client instead. :notOldCouchbase @REM Get the rest of the arguments, skipping the first 2 FOR /F "tokens=2*" %%G IN ("%*") DO ( SET YCSB_ARGS=%%H ) @REM Run YCSB @ECHO ON "%JAVA_HOME%\bin\java.exe" %JAVA_OPTS% -classpath "%CLASSPATH%" %YCSB_CLASS% %YCSB_COMMAND% -db %BINDING_CLASS% %YCSB_ARGS% @ECHO OFF GOTO end :exit EXIT /B 1; :end ================================================ FILE: bin/ycsb.sh ================================================ #!/bin/sh # # Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # ----------------------------------------------------------------------------- # Control Script for YCSB # # Environment Variable Prerequisites # # Do not set the variables in this script. Instead put them into a script # setenv.sh in YCSB_HOME/bin to keep your customizations separate. # # YCSB_HOME (Optional) YCSB installation directory. If not set # this script will use the parent directory of where this # script is run from. # # JAVA_HOME (Optional) Must point at your Java Development Kit # installation. If empty, this script tries use the # available java executable. # # JAVA_OPTS (Optional) Java runtime options used when any command # is executed. # # WARNING!!! YCSB home must be located in a directory path that doesn't # contain spaces. # # www.shellcheck.net was used to validate this script # Cygwin support CYGWIN=false case "$(uname)" in CYGWIN*) CYGWIN=true;; esac # Get script path SCRIPT_DIR=$(dirname "$0" 2>/dev/null) # Only set YCSB_HOME if not already set [ -z "$YCSB_HOME" ] && YCSB_HOME=$(cd "$SCRIPT_DIR/.." || exit; pwd) # Ensure that any extra CLASSPATH variables are set via setenv.sh CLASSPATH= # Pull in customization options if [ -r "$YCSB_HOME/bin/setenv.sh" ]; then # Shellcheck wants a source, but this directive only runs if available # So, tell shellcheck to ignore # shellcheck source=/dev/null . "$YCSB_HOME/bin/setenv.sh" fi # Attempt to find the available JAVA, if JAVA_HOME not set if [ -z "$JAVA_HOME" ]; then JAVA_PATH=$(which java 2>/dev/null) if [ "x$JAVA_PATH" != "x" ]; then JAVA_HOME=$(dirname "$(dirname "$JAVA_PATH" 2>/dev/null)") fi fi # If JAVA_HOME still not set, error if [ -z "$JAVA_HOME" ]; then echo "[ERROR] Java executable not found. Exiting." exit 1; fi # Determine YCSB command argument if [ "load" = "$1" ] ; then YCSB_COMMAND=-load YCSB_CLASS=com.yahoo.ycsb.Client elif [ "run" = "$1" ] ; then YCSB_COMMAND=-t YCSB_CLASS=com.yahoo.ycsb.Client elif [ "shell" = "$1" ] ; then YCSB_COMMAND= YCSB_CLASS=com.yahoo.ycsb.CommandLine else echo "[ERROR] Found unknown command '$1'" echo "[ERROR] Expected one of 'load', 'run', or 'shell'. Exiting." exit 1; fi # Find binding information BINDING_LINE=$(grep "^$2:" "$YCSB_HOME/bin/bindings.properties" -m 1) if [ -z "$BINDING_LINE" ] ; then echo "[ERROR] The specified binding '$2' was not found. Exiting." exit 1; fi # Get binding name and class BINDING_NAME=$(echo "$BINDING_LINE" | cut -d':' -f1) BINDING_CLASS=$(echo "$BINDING_LINE" | cut -d':' -f2) # Some bindings have multiple versions that are managed in the same directory. # They are noted with a '-' after the binding name. # (e.g. cassandra-7 & cassandra-8) BINDING_DIR=$(echo "$BINDING_NAME" | cut -d'-' -f1) # The 'basic' binding is core functionality if [ "$BINDING_NAME" = "basic" ] ; then BINDING_DIR=core fi # For Cygwin, ensure paths are in UNIX format before anything is touched if $CYGWIN; then [ -n "$JAVA_HOME" ] && JAVA_HOME=$(cygpath --unix "$JAVA_HOME") [ -n "$CLASSPATH" ] && CLASSPATH=$(cygpath --path --unix "$CLASSPATH") fi # Check if source checkout, or release distribution DISTRIBUTION=true if [ -r "$YCSB_HOME/pom.xml" ]; then DISTRIBUTION=false; fi # Add Top level conf to classpath if [ -z "$CLASSPATH" ] ; then CLASSPATH="$YCSB_HOME/conf" else CLASSPATH="$CLASSPATH:$YCSB_HOME/conf" fi # Accumulo deprecation message if [ "${BINDING_DIR}" = "accumulo" ] ; then echo "[WARN] The 'accumulo' client has been deprecated in favor of version \ specific bindings. This name still maps to the binding for \ Accumulo 1.6, which is named 'accumulo-1.6'. This alias will \ be removed in a future YCSB release." BINDING_DIR="accumulo1.6" fi # Cassandra2 deprecation message if [ "${BINDING_DIR}" = "cassandra2" ] ; then echo "[WARN] The 'cassandra2-cql' client has been deprecated. It has been \ renamed to simply 'cassandra-cql'. This alias will be removed in the next \ YCSB release." BINDING_DIR="cassandra" fi # Build classpath # The "if" check after the "for" is because glob may just return the pattern # when no files are found. The "if" makes sure the file is really there. if $DISTRIBUTION; then # Core libraries for f in "$YCSB_HOME"/lib/*.jar ; do if [ -r "$f" ] ; then CLASSPATH="$CLASSPATH:$f" fi done # Database conf dir if [ -r "$YCSB_HOME"/"$BINDING_DIR"-binding/conf ] ; then CLASSPATH="$CLASSPATH:$YCSB_HOME/$BINDING_DIR-binding/conf" fi # Database libraries for f in "$YCSB_HOME"/"$BINDING_DIR"-binding/lib/*.jar ; do if [ -r "$f" ] ; then CLASSPATH="$CLASSPATH:$f" fi done # Source checkout else # Check for some basic libraries to see if the source has been built. for f in "$YCSB_HOME"/"$BINDING_DIR"/target/*.jar ; do # Call mvn to build source checkout. if [ ! -e "$f" ] ; then if [ "$BINDING_NAME" = "basic" ] ; then MVN_PROJECT=core else MVN_PROJECT="$BINDING_DIR"-binding fi echo "[WARN] YCSB libraries not found. Attempting to build..." mvn -pl com.yahoo.ycsb:"$MVN_PROJECT" -am package -DskipTests if [ "$?" -ne 0 ] ; then echo "[ERROR] Error trying to build project. Exiting." exit 1; fi fi done # Core libraries for f in "$YCSB_HOME"/core/target/*.jar ; do if [ -r "$f" ] ; then CLASSPATH="$CLASSPATH:$f" fi done # Database conf (need to find because location is not consistent) CLASSPATH_CONF=$(find "$YCSB_HOME"/$BINDING_DIR -name "conf" | while IFS="" read -r file; do echo ":$file"; done) if [ "x$CLASSPATH_CONF" != "x" ]; then CLASSPATH="$CLASSPATH$CLASSPATH_CONF" fi # Database libraries for f in "$YCSB_HOME"/"$BINDING_DIR"/target/*.jar ; do if [ -r "$f" ] ; then CLASSPATH="$CLASSPATH:$f" fi done # Database dependency libraries for f in "$YCSB_HOME"/"$BINDING_DIR"/target/dependency/*.jar ; do if [ -r "$f" ] ; then CLASSPATH="$CLASSPATH:$f" fi done fi # Couchbase deprecation message if [ "${BINDING_DIR}" = "couchbase" ] ; then echo "[WARN] The 'couchbase' client is deprecated. If you are using \ Couchbase 4.0+ try using the 'couchbase2' client instead." fi # For Cygwin, switch paths to Windows format before running java if $CYGWIN; then [ -n "$JAVA_HOME" ] && JAVA_HOME=$(cygpath --unix "$JAVA_HOME") [ -n "$CLASSPATH" ] && CLASSPATH=$(cygpath --path --windows "$CLASSPATH") fi # Get the rest of the arguments YCSB_ARGS=$(echo "$@" | cut -d' ' -f3-) # About to run YCSB echo "$JAVA_HOME/bin/java $JAVA_OPTS -classpath $CLASSPATH $YCSB_CLASS $YCSB_COMMAND -db $BINDING_CLASS $YCSB_ARGS" # Run YCSB # Shellcheck reports the following line as needing double quotes to prevent # globbing and word splitting. However, word splitting is the desired effect # here. So, the shellcheck error is disabled for this line. # shellcheck disable=SC2086 "$JAVA_HOME/bin/java" $JAVA_OPTS -classpath "$CLASSPATH" $YCSB_CLASS $YCSB_COMMAND -db $BINDING_CLASS $YCSB_ARGS ================================================ FILE: binding-parent/datastore-specific-descriptor/pom.xml ================================================ 4.0.0 com.yahoo.ycsb root 0.14.0-SNAPSHOT ../../ datastore-specific-descriptor Per Datastore Binding descriptor jar This module contains the assembly descriptor used by the individual components to build binding-specific distributions. com.yahoo.ycsb core ${project.version} ================================================ FILE: binding-parent/datastore-specific-descriptor/src/main/resources/assemblies/datastore-specific-assembly.xml ================================================ dist true ycsb-${artifactId}-${version} README.md .. 0644 LICENSE.txt NOTICE.txt ../bin bin 0755 ycsb* ../bin bin 0644 bindings.properties ../workloads workloads 0644 src/main/conf conf 0644 lib com.yahoo.ycsb:core provided true lib *:jar:* *:sources ================================================ FILE: binding-parent/pom.xml ================================================ 4.0.0 com.yahoo.ycsb root 0.14.0-SNAPSHOT binding-parent YCSB Datastore Binding Parent pom This module acts as the parent for new datastore bindings. It creates a datastore specific binary artifact. datastore-specific-descriptor false org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} com.yahoo.ycsb datastore-specific-descriptor ${project.version} datastore-specific-assembly ycsb-${project.artifactId}-${project.version} tar.gz false posix package single org.apache.maven.plugins maven-checkstyle-plugin validate ../checkstyle.xml org.apache.maven.plugins maven-dependency-plugin ${maven.dependency.version} org.apache.maven.plugins maven-dependency-plugin stage-dependencies package copy-dependencies runtime datastore-binding README.md org.apache.maven.plugins maven-assembly-plugin tests-on-jdk9 9 ${skipJDK9Tests} ================================================ FILE: cassandra/README.md ================================================ # Apache Cassandra 2.x CQL binding Binding for [Apache Cassandra](http://cassandra.apache.org), using the CQL API via the [DataStax driver](http://docs.datastax.com/en/developer/java-driver/2.1/java-driver/whatsNew2.html). To run against the (deprecated) Cassandra Thrift API, use the `cassandra-10` binding. ## Creating a table for use with YCSB For keyspace `ycsb`, table `usertable`: cqlsh> create keyspace ycsb WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor': 3 }; cqlsh> USE ycsb; cqlsh> create table usertable ( y_id varchar primary key, field0 varchar, field1 varchar, field2 varchar, field3 varchar, field4 varchar, field5 varchar, field6 varchar, field7 varchar, field8 varchar, field9 varchar); **Note that `replication_factor` and consistency levels (below) will affect performance.** ## Cassandra Configuration Parameters - `hosts` (**required**) - Cassandra nodes to connect to. - No default. * `port` * CQL port for communicating with Cassandra cluster. * Default is `9042`. - `cassandra.keyspace` Keyspace name - must match the keyspace for the table created (see above). See http://docs.datastax.com/en/cql/3.1/cql/cql_reference/create_keyspace_r.html for details. - Default value is `ycsb` - `cassandra.username` - `cassandra.password` - Optional user name and password for authentication. See http://docs.datastax.com/en/cassandra/2.0/cassandra/security/security_config_native_authenticate_t.html for details. * `cassandra.readconsistencylevel` * `cassandra.writeconsistencylevel` * Default value is `ONE` - Consistency level for reads and writes, respectively. See the [DataStax documentation](http://docs.datastax.com/en/cassandra/2.0/cassandra/dml/dml_config_consistency_c.html) for details. * *Note that the default setting does not provide durability in the face of node failure. Changing this setting will affect observed performance.* See also `replication_factor`, above. * `cassandra.maxconnections` * `cassandra.coreconnections` * Defaults for max and core connections can be found here: https://datastax.github.io/java-driver/2.1.8/features/pooling/#pool-size. Cassandra 2.0.X falls under protocol V2, Cassandra 2.1+ falls under protocol V3. * `cassandra.connecttimeoutmillis` * `cassandra.readtimeoutmillis` * Defaults for connect and read timeouts can be found here: https://docs.datastax.com/en/drivers/java/2.0/com/datastax/driver/core/SocketOptions.html. * `cassandra.tracing` * Default is false * https://docs.datastax.com/en/cql/3.3/cql/cql_reference/tracing_r.html ================================================ FILE: cassandra/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent cassandra-binding Cassandra 2.1+ DB Binding jar true com.datastax.cassandra cassandra-driver-core ${cassandra.cql.version} com.yahoo.ycsb core ${project.version} provided org.cassandraunit cassandra-unit 3.0.0.1 shaded test org.slf4j slf4j-simple 1.7.21 test junit junit 4.12 test org.hyperic sigar-dist 1.6.4.129 zip test jdk8-tests 1.8 false central2 sigar Repository http://repository.jboss.org/nexus/content/groups/public-jboss/ default false org.apache.maven.plugins maven-dependency-plugin unpack-sigar process-test-resources unpack-dependencies org.hyperic sigar-dist **/sigar-bin/lib/* **/sigar-bin/lib/*jar ${project.build.directory}/cassandra-dependency org.apache.maven.plugins maven-surefire-plugin 2.8 -Djava.library.path=${project.build.directory}/cassandra-dependency/hyperic-sigar-1.6.4/sigar-bin/lib ================================================ FILE: cassandra/src/main/java/com/yahoo/ycsb/db/CassandraCQLClient.java ================================================ /** * Copyright (c) 2013-2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. See accompanying LICENSE file. * * Submitted by Chrisjan Matser on 10/11/2010. */ package com.yahoo.ycsb.db; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ColumnDefinitions; import com.datastax.driver.core.ConsistencyLevel; import com.datastax.driver.core.Host; import com.datastax.driver.core.HostDistance; import com.datastax.driver.core.Metadata; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.datastax.driver.core.SimpleStatement; import com.datastax.driver.core.Statement; import com.datastax.driver.core.querybuilder.Insert; import com.datastax.driver.core.querybuilder.QueryBuilder; import com.datastax.driver.core.querybuilder.Select; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; /** * Cassandra 2.x CQL client. * * See {@code cassandra2/README.md} for details. * * @author cmatser */ public class CassandraCQLClient extends DB { private static Cluster cluster = null; private static Session session = null; private static ConsistencyLevel readConsistencyLevel = ConsistencyLevel.ONE; private static ConsistencyLevel writeConsistencyLevel = ConsistencyLevel.ONE; public static final String YCSB_KEY = "y_id"; public static final String KEYSPACE_PROPERTY = "cassandra.keyspace"; public static final String KEYSPACE_PROPERTY_DEFAULT = "ycsb"; public static final String USERNAME_PROPERTY = "cassandra.username"; public static final String PASSWORD_PROPERTY = "cassandra.password"; public static final String HOSTS_PROPERTY = "hosts"; public static final String PORT_PROPERTY = "port"; public static final String PORT_PROPERTY_DEFAULT = "9042"; public static final String READ_CONSISTENCY_LEVEL_PROPERTY = "cassandra.readconsistencylevel"; public static final String READ_CONSISTENCY_LEVEL_PROPERTY_DEFAULT = "ONE"; public static final String WRITE_CONSISTENCY_LEVEL_PROPERTY = "cassandra.writeconsistencylevel"; public static final String WRITE_CONSISTENCY_LEVEL_PROPERTY_DEFAULT = "ONE"; public static final String MAX_CONNECTIONS_PROPERTY = "cassandra.maxconnections"; public static final String CORE_CONNECTIONS_PROPERTY = "cassandra.coreconnections"; public static final String CONNECT_TIMEOUT_MILLIS_PROPERTY = "cassandra.connecttimeoutmillis"; public static final String READ_TIMEOUT_MILLIS_PROPERTY = "cassandra.readtimeoutmillis"; public static final String TRACING_PROPERTY = "cassandra.tracing"; public static final String TRACING_PROPERTY_DEFAULT = "false"; /** * Count the number of times initialized to teardown on the last * {@link #cleanup()}. */ private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); private static boolean debug = false; private static boolean trace = false; /** * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void init() throws DBException { // Keep track of number of calls to init (for later cleanup) INIT_COUNT.incrementAndGet(); // Synchronized so that we only have a single // cluster/session instance for all the threads. synchronized (INIT_COUNT) { // Check if the cluster has already been initialized if (cluster != null) { return; } try { debug = Boolean.parseBoolean(getProperties().getProperty("debug", "false")); trace = Boolean.valueOf(getProperties().getProperty(TRACING_PROPERTY, TRACING_PROPERTY_DEFAULT)); String host = getProperties().getProperty(HOSTS_PROPERTY); if (host == null) { throw new DBException(String.format( "Required property \"%s\" missing for CassandraCQLClient", HOSTS_PROPERTY)); } String[] hosts = host.split(","); String port = getProperties().getProperty(PORT_PROPERTY, PORT_PROPERTY_DEFAULT); String username = getProperties().getProperty(USERNAME_PROPERTY); String password = getProperties().getProperty(PASSWORD_PROPERTY); String keyspace = getProperties().getProperty(KEYSPACE_PROPERTY, KEYSPACE_PROPERTY_DEFAULT); readConsistencyLevel = ConsistencyLevel.valueOf( getProperties().getProperty(READ_CONSISTENCY_LEVEL_PROPERTY, READ_CONSISTENCY_LEVEL_PROPERTY_DEFAULT)); writeConsistencyLevel = ConsistencyLevel.valueOf( getProperties().getProperty(WRITE_CONSISTENCY_LEVEL_PROPERTY, WRITE_CONSISTENCY_LEVEL_PROPERTY_DEFAULT)); if ((username != null) && !username.isEmpty()) { cluster = Cluster.builder().withCredentials(username, password) .withPort(Integer.valueOf(port)).addContactPoints(hosts).build(); } else { cluster = Cluster.builder().withPort(Integer.valueOf(port)) .addContactPoints(hosts).build(); } String maxConnections = getProperties().getProperty( MAX_CONNECTIONS_PROPERTY); if (maxConnections != null) { cluster.getConfiguration().getPoolingOptions() .setMaxConnectionsPerHost(HostDistance.LOCAL, Integer.valueOf(maxConnections)); } String coreConnections = getProperties().getProperty( CORE_CONNECTIONS_PROPERTY); if (coreConnections != null) { cluster.getConfiguration().getPoolingOptions() .setCoreConnectionsPerHost(HostDistance.LOCAL, Integer.valueOf(coreConnections)); } String connectTimoutMillis = getProperties().getProperty( CONNECT_TIMEOUT_MILLIS_PROPERTY); if (connectTimoutMillis != null) { cluster.getConfiguration().getSocketOptions() .setConnectTimeoutMillis(Integer.valueOf(connectTimoutMillis)); } String readTimoutMillis = getProperties().getProperty( READ_TIMEOUT_MILLIS_PROPERTY); if (readTimoutMillis != null) { cluster.getConfiguration().getSocketOptions() .setReadTimeoutMillis(Integer.valueOf(readTimoutMillis)); } Metadata metadata = cluster.getMetadata(); System.err.printf("Connected to cluster: %s\n", metadata.getClusterName()); for (Host discoveredHost : metadata.getAllHosts()) { System.out.printf("Datacenter: %s; Host: %s; Rack: %s\n", discoveredHost.getDatacenter(), discoveredHost.getAddress(), discoveredHost.getRack()); } session = cluster.connect(keyspace); } catch (Exception e) { throw new DBException(e); } } // synchronized } /** * Cleanup any state for this DB. Called once per DB instance; there is one DB * instance per client thread. */ @Override public void cleanup() throws DBException { synchronized (INIT_COUNT) { final int curInitCount = INIT_COUNT.decrementAndGet(); if (curInitCount <= 0) { session.close(); cluster.close(); cluster = null; session = null; } if (curInitCount < 0) { // This should never happen. throw new DBException( String.format("initCount is negative: %d", curInitCount)); } } } /** * Read a record from the database. Each field/value pair from the result will * be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ @Override public Status read(String table, String key, Set fields, Map result) { try { Statement stmt; Select.Builder selectBuilder; if (fields == null) { selectBuilder = QueryBuilder.select().all(); } else { selectBuilder = QueryBuilder.select(); for (String col : fields) { ((Select.Selection) selectBuilder).column(col); } } stmt = selectBuilder.from(table).where(QueryBuilder.eq(YCSB_KEY, key)) .limit(1); stmt.setConsistencyLevel(readConsistencyLevel); if (debug) { System.out.println(stmt.toString()); } if (trace) { stmt.enableTracing(); } ResultSet rs = session.execute(stmt); if (rs.isExhausted()) { return Status.NOT_FOUND; } // Should be only 1 row Row row = rs.one(); ColumnDefinitions cd = row.getColumnDefinitions(); for (ColumnDefinitions.Definition def : cd) { ByteBuffer val = row.getBytesUnsafe(def.getName()); if (val != null) { result.put(def.getName(), new ByteArrayByteIterator(val.array())); } else { result.put(def.getName(), null); } } return Status.OK; } catch (Exception e) { e.printStackTrace(); System.out.println("Error reading key: " + key); return Status.ERROR; } } /** * Perform a range scan for a set of records in the database. Each field/value * pair from the result will be stored in a HashMap. * * Cassandra CQL uses "token" method for range scan which doesn't always yield * intuitive results. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { Statement stmt; Select.Builder selectBuilder; if (fields == null) { selectBuilder = QueryBuilder.select().all(); } else { selectBuilder = QueryBuilder.select(); for (String col : fields) { ((Select.Selection) selectBuilder).column(col); } } stmt = selectBuilder.from(table); // The statement builder is not setup right for tokens. // So, we need to build it manually. String initialStmt = stmt.toString(); StringBuilder scanStmt = new StringBuilder(); scanStmt.append(initialStmt.substring(0, initialStmt.length() - 1)); scanStmt.append(" WHERE "); scanStmt.append(QueryBuilder.token(YCSB_KEY)); scanStmt.append(" >= "); scanStmt.append("token('"); scanStmt.append(startkey); scanStmt.append("')"); scanStmt.append(" LIMIT "); scanStmt.append(recordcount); stmt = new SimpleStatement(scanStmt.toString()); stmt.setConsistencyLevel(readConsistencyLevel); if (debug) { System.out.println(stmt.toString()); } if (trace) { stmt.enableTracing(); } ResultSet rs = session.execute(stmt); HashMap tuple; while (!rs.isExhausted()) { Row row = rs.one(); tuple = new HashMap(); ColumnDefinitions cd = row.getColumnDefinitions(); for (ColumnDefinitions.Definition def : cd) { ByteBuffer val = row.getBytesUnsafe(def.getName()); if (val != null) { tuple.put(def.getName(), new ByteArrayByteIterator(val.array())); } else { tuple.put(def.getName(), null); } } result.add(tuple); } return Status.OK; } catch (Exception e) { e.printStackTrace(); System.out.println("Error scanning with startkey: " + startkey); return Status.ERROR; } } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error */ @Override public Status update(String table, String key, Map values) { // Insert and updates provide the same functionality return insert(table, key, values); } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error */ @Override public Status insert(String table, String key, Map values) { try { Insert insertStmt = QueryBuilder.insertInto(table); // Add key insertStmt.value(YCSB_KEY, key); // Add fields for (Map.Entry entry : values.entrySet()) { Object value; ByteIterator byteIterator = entry.getValue(); value = byteIterator.toString(); insertStmt.value(entry.getKey(), value); } insertStmt.setConsistencyLevel(writeConsistencyLevel); if (debug) { System.out.println(insertStmt.toString()); } if (trace) { insertStmt.enableTracing(); } session.execute(insertStmt); return Status.OK; } catch (Exception e) { e.printStackTrace(); } return Status.ERROR; } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error */ @Override public Status delete(String table, String key) { try { Statement stmt; stmt = QueryBuilder.delete().from(table) .where(QueryBuilder.eq(YCSB_KEY, key)); stmt.setConsistencyLevel(writeConsistencyLevel); if (debug) { System.out.println(stmt.toString()); } if (trace) { stmt.enableTracing(); } session.execute(stmt); return Status.OK; } catch (Exception e) { e.printStackTrace(); System.out.println("Error deleting key: " + key); } return Status.ERROR; } } ================================================ FILE: cassandra/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Cassandra * 2.1+ via CQL. */ package com.yahoo.ycsb.db; ================================================ FILE: cassandra/src/test/java/com/yahoo/ycsb/db/CassandraCQLClientTest.java ================================================ /** * Copyright (c) 2015 YCSB contributors All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import com.google.common.collect.Sets; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.datastax.driver.core.Statement; import com.datastax.driver.core.querybuilder.Insert; import com.datastax.driver.core.querybuilder.QueryBuilder; import com.datastax.driver.core.querybuilder.Select; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.workloads.CoreWorkload; import org.cassandraunit.CassandraCQLUnit; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; /** * Integration tests for the Cassandra client */ public class CassandraCQLClientTest { // Change the default Cassandra timeout from 10s to 120s for slow CI machines private final static long timeout = 120000L; private final static String TABLE = "usertable"; private final static String HOST = "localhost"; private final static int PORT = 9142; private final static String DEFAULT_ROW_KEY = "user1"; private CassandraCQLClient client; private Session session; @ClassRule public static CassandraCQLUnit cassandraUnit = new CassandraCQLUnit( new ClassPathCQLDataSet("ycsb.cql", "ycsb"), null, timeout); @Before public void setUp() throws Exception { session = cassandraUnit.getSession(); Properties p = new Properties(); p.setProperty("hosts", HOST); p.setProperty("port", Integer.toString(PORT)); p.setProperty("table", TABLE); Measurements.setProperties(p); final CoreWorkload workload = new CoreWorkload(); workload.init(p); client = new CassandraCQLClient(); client.setProperties(p); client.init(); } @After public void tearDownClient() throws Exception { if (client != null) { client.cleanup(); } client = null; } @After public void clearTable() throws Exception { // Clear the table so that each test starts fresh. final Statement truncate = QueryBuilder.truncate(TABLE); if (cassandraUnit != null) { cassandraUnit.getSession().execute(truncate); } } @Test public void testReadMissingRow() throws Exception { final HashMap result = new HashMap(); final Status status = client.read(TABLE, "Missing row", null, result); assertThat(result.size(), is(0)); assertThat(status, is(Status.NOT_FOUND)); } private void insertRow() { final String rowKey = DEFAULT_ROW_KEY; Insert insertStmt = QueryBuilder.insertInto(TABLE); insertStmt.value(CassandraCQLClient.YCSB_KEY, rowKey); insertStmt.value("field0", "value1"); insertStmt.value("field1", "value2"); session.execute(insertStmt); } @Test public void testRead() throws Exception { insertRow(); final HashMap result = new HashMap(); final Status status = client.read(TABLE, DEFAULT_ROW_KEY, null, result); assertThat(status, is(Status.OK)); assertThat(result.entrySet(), hasSize(11)); assertThat(result, hasEntry("field2", null)); final HashMap strResult = new HashMap(); for (final Map.Entry e : result.entrySet()) { if (e.getValue() != null) { strResult.put(e.getKey(), e.getValue().toString()); } } assertThat(strResult, hasEntry(CassandraCQLClient.YCSB_KEY, DEFAULT_ROW_KEY)); assertThat(strResult, hasEntry("field0", "value1")); assertThat(strResult, hasEntry("field1", "value2")); } @Test public void testReadSingleColumn() throws Exception { insertRow(); final HashMap result = new HashMap(); final Set fields = Sets.newHashSet("field1"); final Status status = client.read(TABLE, DEFAULT_ROW_KEY, fields, result); assertThat(status, is(Status.OK)); assertThat(result.entrySet(), hasSize(1)); final Map strResult = StringByteIterator.getStringMap(result); assertThat(strResult, hasEntry("field1", "value2")); } @Test public void testUpdate() throws Exception { final String key = "key"; final Map input = new HashMap(); input.put("field0", "value1"); input.put("field1", "value2"); final Status status = client.insert(TABLE, key, StringByteIterator.getByteIteratorMap(input)); assertThat(status, is(Status.OK)); // Verify result final Select selectStmt = QueryBuilder.select("field0", "field1") .from(TABLE) .where(QueryBuilder.eq(CassandraCQLClient.YCSB_KEY, key)) .limit(1); final ResultSet rs = session.execute(selectStmt); final Row row = rs.one(); assertThat(row, notNullValue()); assertThat(rs.isExhausted(), is(true)); assertThat(row.getString("field0"), is("value1")); assertThat(row.getString("field1"), is("value2")); } } ================================================ FILE: cassandra/src/test/resources/ycsb.cql ================================================ /** * Copyright (c) 2015 YCSB Contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ CREATE TABLE usertable ( y_id varchar primary key, field0 varchar, field1 varchar, field2 varchar, field3 varchar, field4 varchar, field5 varchar, field6 varchar, field7 varchar, field8 varchar, field9 varchar); ================================================ FILE: checkstyle.xml ================================================ ================================================ FILE: cloudspanner/README.md ================================================ # Cloud Spanner Driver for YCSB This driver provides a YCSB workload binding for Google's Cloud Spanner database, the first relational database service that is both strongly consistent and horizontally scalable. This binding is implemented using the official Java client library for Cloud Spanner which uses GRPC for making calls. For best results, we strongly recommend running the benchmark from a Google Compute Engine (GCE) VM. ## Running a Workload We recommend reading the [general guidelines](https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload) in the YCSB documentation, and following the Cloud Spanner specific steps below. ### 1. Set up Cloud Spanner with the Expected Schema Follow the [Quickstart instructions](https://cloud.google.com/spanner/docs/quickstart-console) in the Cloud Spanner documentation to set up a Cloud Spanner instance, and create a database with the following schema: ``` CREATE TABLE usertable ( id STRING(MAX), field0 STRING(MAX), field1 STRING(MAX), field2 STRING(MAX), field3 STRING(MAX), field4 STRING(MAX), field5 STRING(MAX), field6 STRING(MAX), field7 STRING(MAX), field8 STRING(MAX), field9 STRING(MAX), ) PRIMARY KEY(id); ``` Make note of your project ID, instance ID, and database name. ### 2. Set Up Your Environment and Auth Follow the [set up instructions](https://cloud.google.com/spanner/docs/getting-started/set-up) in the Cloud Spanner documentation to set up your environment and authentication. When not running on a GCE VM, make sure you run `gcloud auth application-default login`. ### 3. Edit Properties In your YCSB root directory, edit `cloudspanner/conf/cloudspanner.properties` and specify your project ID, instance ID, and database name. ### 4. Run the YCSB Shell Start the YCBS shell connected to Cloud Spanner using the following command: ``` ./bin/ycsb shell cloudspanner -P cloudspanner/conf/cloudspanner.properties ``` You can use the `insert`, `read`, `update`, `scan`, and `delete` commands in the shell to experiment with your database and make sure the connection works. For example, try the following: ``` insert name field0=adam read name field0 delete name ``` ### 5. Load the Data You can load, say, 10 GB of data into your YCSB database using the following command: ``` ./bin/ycsb load cloudspanner -P cloudspanner/conf/cloudspanner.properties -P workloads/workloada -p recordcount=10000000 -p cloudspanner.batchinserts=1000 -threads 10 -s ``` We recommend batching insertions so as to reach ~1 MB of data per commit request; this is controlled via the `cloudspanner.batchinserts` parameter which we recommend setting to `1000` during data load. If you wish to load a large database, you can run YCSB on multiple client VMs in parallel and use the `insertstart` and `insertcount` parameters to distribute the load as described [here](https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload-in-Parallel). In this case, we recommend the following: * Use ordered inserts via specifying the YCSB parameter `insertorder=ordered`; * Use zero-padding so that ordered inserts are actually lexicographically ordered; the option `zeropadding = 12` is set in the default `cloudspanner.properties` file; * Split the key range evenly between client VMs; * Use few threads on each client VM, so that each individual commit request contains keys which are (close to) consecutive, and would thus likely address a single split; this also helps avoid overloading the servers. The idea is that we have a number of 'write heads' which are all writing to different parts of the database (and thus talking to different servers), but each individual head is writing its own data (more or less) in order. See the [best practices page](https://cloud.google.com/spanner/docs/best-practices#loading_data) for further details. ### 6. Run a Workload After data load, you can a run a workload, say, workload B, using the following command: ``` ./bin/ycsb run cloudspanner -P cloudspanner/conf/cloudspanner.properties -P workloads/workloadb -p recordcount=10000000 -p operationcount=1000000 -threads 10 -s ``` Make sure that you use the same `insertorder` (i.e. `ordered` or `hashed`) and `zeropadding` as specified during the data load. Further details about running workloads are given in the [YCSB wiki pages](https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload). ## Configuration Options In addition to the standard YCSB parameters, the following Cloud Spanner specific options can be configured using the `-p` parameter or in `cloudspanner/conf/cloudspanner.properties`. * `cloudspanner.database`: (Required) The name of the database created in the instance, e.g. `ycsb-database`. * `cloudspanner.instance`: (Required) The ID of the Cloud Spanner instance, e.g. `ycsb-instance`. * `cloudspanner.project`: The ID of the project containing the Cloud Spanner instance, e.g. `myproject`. This is not strictly required and can often be automatically inferred from the environment. * `cloudspanner.readmode`: Allows choosing between the `read` and `query` interface of Cloud Spanner. The default is `query`. * `cloudspanner.batchinserts`: The number of inserts to batch into a single commit request. The default value is 1 which means no batching is done. Recommended value during data load is 1000. * `cloudspanner.boundedstaleness`: Number of seconds we allow reads to be stale for. Set to 0 for strong reads (default). For performance gains, this should be set to 10 seconds. ================================================ FILE: cloudspanner/conf/cloudspanner.properties ================================================ # Copyright (c) 2017 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Core YCSB properties. table = usertable zeropadding = 12 # Cloud Spanner properties cloudspanner.instance = ycsb-instance cloudspanner.database = ycsb-database cloudspanner.readmode = query cloudspanner.boundedstaleness = 0 cloudspanner.batchinserts = 1 ================================================ FILE: cloudspanner/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ cloudspanner-binding Cloud Spanner DB Binding jar com.google.cloud google-cloud-spanner ${cloudspanner.version} com.google.guava guava-jdk5 com.yahoo.ycsb core ${project.version} provided ================================================ FILE: cloudspanner/src/main/java/com/yahoo/ycsb/db/cloudspanner/CloudSpannerClient.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.cloudspanner; import com.google.common.base.Joiner; import com.google.cloud.spanner.DatabaseId; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.Key; import com.google.cloud.spanner.KeySet; import com.google.cloud.spanner.KeyRange; import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.Options; import com.google.cloud.spanner.ResultSet; import com.google.cloud.spanner.SessionPoolOptions; import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.Statement; import com.google.cloud.spanner.Struct; import com.google.cloud.spanner.StructReader; import com.google.cloud.spanner.TimestampBound; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Client; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.workloads.CoreWorkload; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.logging.Level; import java.util.logging.Logger; import java.util.concurrent.TimeUnit; /** * YCSB Client for Google's Cloud Spanner. */ public class CloudSpannerClient extends DB { /** * The names of properties which can be specified in the config files and flags. */ public static final class CloudSpannerProperties { private CloudSpannerProperties() {} /** * The Cloud Spanner database name to use when running the YCSB benchmark, e.g. 'ycsb-database'. */ static final String DATABASE = "cloudspanner.database"; /** * The Cloud Spanner instance ID to use when running the YCSB benchmark, e.g. 'ycsb-instance'. */ static final String INSTANCE = "cloudspanner.instance"; /** * Choose between 'read' and 'query'. Affects both read() and scan() operations. */ static final String READ_MODE = "cloudspanner.readmode"; /** * The number of inserts to batch during the bulk loading phase. The default value is 1, which means no batching * is done. Recommended value during data load is 1000. */ static final String BATCH_INSERTS = "cloudspanner.batchinserts"; /** * Number of seconds we allow reads to be stale for. Set to 0 for strong reads (default). * For performance gains, this should be set to 10 seconds. */ static final String BOUNDED_STALENESS = "cloudspanner.boundedstaleness"; // The properties below usually do not need to be set explicitly. /** * The Cloud Spanner project ID to use when running the YCSB benchmark, e.g. 'myproject'. This is not strictly * necessary and can often be inferred from the environment. */ static final String PROJECT = "cloudspanner.project"; /** * The Cloud Spanner host name to use in the YCSB run. */ static final String HOST = "cloudspanner.host"; /** * Number of Cloud Spanner client channels to use. It's recommended to leave this to be the default value. */ static final String NUM_CHANNELS = "cloudspanner.channels"; } private static int fieldCount; private static boolean queriesForReads; private static int batchInserts; private static TimestampBound timestampBound; private static String standardQuery; private static String standardScan; private static final ArrayList STANDARD_FIELDS = new ArrayList<>(); private static final String PRIMARY_KEY_COLUMN = "id"; private static final Logger LOGGER = Logger.getLogger(CloudSpannerClient.class.getName()); // Static lock for the class. private static final Object CLASS_LOCK = new Object(); // Single Spanner client per process. private static Spanner spanner = null; // Single database client per process. private static DatabaseClient dbClient = null; // Buffered mutations on a per object/thread basis for batch inserts. // Note that we have a separate CloudSpannerClient object per thread. private final ArrayList bufferedMutations = new ArrayList<>(); private static void constructStandardQueriesAndFields(Properties properties) { String table = properties.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT); standardQuery = new StringBuilder() .append("SELECT * FROM ").append(table).append(" WHERE id=@key").toString(); standardScan = new StringBuilder() .append("SELECT * FROM ").append(table).append(" WHERE id>=@startKey LIMIT @count").toString(); for (int i = 0; i < fieldCount; i++) { STANDARD_FIELDS.add("field" + i); } } private static Spanner getSpanner(Properties properties, String host, String project) { if (spanner != null) { return spanner; } String numChannels = properties.getProperty(CloudSpannerProperties.NUM_CHANNELS); int numThreads = Integer.parseInt(properties.getProperty(Client.THREAD_COUNT_PROPERTY, "1")); SpannerOptions.Builder optionsBuilder = SpannerOptions.newBuilder() .setSessionPoolOption(SessionPoolOptions.newBuilder() .setMinSessions(numThreads) // Since we have no read-write transactions, we can set the write session fraction to 0. .setWriteSessionsFraction(0) .build()); if (host != null) { optionsBuilder.setHost(host); } if (project != null) { optionsBuilder.setProjectId(project); } if (numChannels != null) { optionsBuilder.setNumChannels(Integer.parseInt(numChannels)); } spanner = optionsBuilder.build().getService(); Runtime.getRuntime().addShutdownHook(new Thread("spannerShutdown") { @Override public void run() { spanner.close(); } }); return spanner; } @Override public void init() throws DBException { synchronized (CLASS_LOCK) { if (dbClient != null) { return; } Properties properties = getProperties(); String host = properties.getProperty(CloudSpannerProperties.HOST); String project = properties.getProperty(CloudSpannerProperties.PROJECT); String instance = properties.getProperty(CloudSpannerProperties.INSTANCE, "ycsb-instance"); String database = properties.getProperty(CloudSpannerProperties.DATABASE, "ycsb-database"); fieldCount = Integer.parseInt(properties.getProperty( CoreWorkload.FIELD_COUNT_PROPERTY, CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT)); queriesForReads = properties.getProperty(CloudSpannerProperties.READ_MODE, "query").equals("query"); batchInserts = Integer.parseInt(properties.getProperty(CloudSpannerProperties.BATCH_INSERTS, "1")); constructStandardQueriesAndFields(properties); int boundedStalenessSeconds = Integer.parseInt(properties.getProperty( CloudSpannerProperties.BOUNDED_STALENESS, "0")); timestampBound = (boundedStalenessSeconds <= 0) ? TimestampBound.strong() : TimestampBound.ofMaxStaleness(boundedStalenessSeconds, TimeUnit.SECONDS); try { spanner = getSpanner(properties, host, project); if (project == null) { project = spanner.getOptions().getProjectId(); } dbClient = spanner.getDatabaseClient(DatabaseId.of(project, instance, database)); } catch (Exception e) { LOGGER.log(Level.SEVERE, "init()", e); throw new DBException(e); } LOGGER.log(Level.INFO, new StringBuilder() .append("\nHost: ").append(spanner.getOptions().getHost()) .append("\nProject: ").append(project) .append("\nInstance: ").append(instance) .append("\nDatabase: ").append(database) .append("\nUsing queries for reads: ").append(queriesForReads) .append("\nBatching inserts: ").append(batchInserts) .append("\nBounded staleness seconds: ").append(boundedStalenessSeconds) .toString()); } } private Status readUsingQuery( String table, String key, Set fields, Map result) { Statement query; Iterable columns = fields == null ? STANDARD_FIELDS : fields; if (fields == null || fields.size() == fieldCount) { query = Statement.newBuilder(standardQuery).bind("key").to(key).build(); } else { Joiner joiner = Joiner.on(','); query = Statement.newBuilder("SELECT ") .append(joiner.join(fields)) .append(" FROM ") .append(table) .append(" WHERE id=@key") .bind("key").to(key) .build(); } try (ResultSet resultSet = dbClient.singleUse(timestampBound).executeQuery(query)) { resultSet.next(); decodeStruct(columns, resultSet, result); if (resultSet.next()) { throw new Exception("Expected exactly one row for each read."); } return Status.OK; } catch (Exception e) { LOGGER.log(Level.INFO, "readUsingQuery()", e); return Status.ERROR; } } @Override public Status read( String table, String key, Set fields, Map result) { if (queriesForReads) { return readUsingQuery(table, key, fields, result); } Iterable columns = fields == null ? STANDARD_FIELDS : fields; try { Struct row = dbClient.singleUse(timestampBound).readRow(table, Key.of(key), columns); decodeStruct(columns, row, result); return Status.OK; } catch (Exception e) { LOGGER.log(Level.INFO, "read()", e); return Status.ERROR; } } private Status scanUsingQuery( String table, String startKey, int recordCount, Set fields, Vector> result) { Iterable columns = fields == null ? STANDARD_FIELDS : fields; Statement query; if (fields == null || fields.size() == fieldCount) { query = Statement.newBuilder(standardScan).bind("startKey").to(startKey).bind("count").to(recordCount).build(); } else { Joiner joiner = Joiner.on(','); query = Statement.newBuilder("SELECT ") .append(joiner.join(fields)) .append(" FROM ") .append(table) .append(" WHERE id>=@startKey LIMIT @count") .bind("startKey").to(startKey) .bind("count").to(recordCount) .build(); } try (ResultSet resultSet = dbClient.singleUse(timestampBound).executeQuery(query)) { while (resultSet.next()) { HashMap row = new HashMap<>(); decodeStruct(columns, resultSet, row); result.add(row); } return Status.OK; } catch (Exception e) { LOGGER.log(Level.INFO, "scanUsingQuery()", e); return Status.ERROR; } } @Override public Status scan( String table, String startKey, int recordCount, Set fields, Vector> result) { if (queriesForReads) { return scanUsingQuery(table, startKey, recordCount, fields, result); } Iterable columns = fields == null ? STANDARD_FIELDS : fields; KeySet keySet = KeySet.newBuilder().addRange(KeyRange.closedClosed(Key.of(startKey), Key.of())).build(); try (ResultSet resultSet = dbClient.singleUse(timestampBound) .read(table, keySet, columns, Options.limit(recordCount))) { while (resultSet.next()) { HashMap row = new HashMap<>(); decodeStruct(columns, resultSet, row); result.add(row); } return Status.OK; } catch (Exception e) { LOGGER.log(Level.INFO, "scan()", e); return Status.ERROR; } } @Override public Status update(String table, String key, Map values) { Mutation.WriteBuilder m = Mutation.newInsertOrUpdateBuilder(table); m.set(PRIMARY_KEY_COLUMN).to(key); for (Map.Entry e : values.entrySet()) { m.set(e.getKey()).to(e.getValue().toString()); } try { dbClient.writeAtLeastOnce(Arrays.asList(m.build())); } catch (Exception e) { LOGGER.log(Level.INFO, "update()", e); return Status.ERROR; } return Status.OK; } @Override public Status insert(String table, String key, Map values) { if (bufferedMutations.size() < batchInserts) { Mutation.WriteBuilder m = Mutation.newInsertOrUpdateBuilder(table); m.set(PRIMARY_KEY_COLUMN).to(key); for (Map.Entry e : values.entrySet()) { m.set(e.getKey()).to(e.getValue().toString()); } bufferedMutations.add(m.build()); } else { LOGGER.log(Level.INFO, "Limit of cached mutations reached. The given mutation with key " + key + " is ignored. Is this a retry?"); } if (bufferedMutations.size() < batchInserts) { return Status.BATCHED_OK; } try { dbClient.writeAtLeastOnce(bufferedMutations); bufferedMutations.clear(); } catch (Exception e) { LOGGER.log(Level.INFO, "insert()", e); return Status.ERROR; } return Status.OK; } @Override public void cleanup() { try { if (bufferedMutations.size() > 0) { dbClient.writeAtLeastOnce(bufferedMutations); bufferedMutations.clear(); } } catch (Exception e) { LOGGER.log(Level.INFO, "cleanup()", e); } } @Override public Status delete(String table, String key) { try { dbClient.writeAtLeastOnce(Arrays.asList(Mutation.delete(table, Key.of(key)))); } catch (Exception e) { LOGGER.log(Level.INFO, "delete()", e); return Status.ERROR; } return Status.OK; } private static void decodeStruct( Iterable columns, StructReader structReader, Map result) { for (String col : columns) { result.put(col, new StringByteIterator(structReader.getString(col))); } } } ================================================ FILE: cloudspanner/src/main/java/com/yahoo/ycsb/db/cloudspanner/package-info.java ================================================ /* * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Google's * Cloud Spanner. */ package com.yahoo.ycsb.db.cloudspanner; ================================================ FILE: core/CHANGES.md ================================================ When used as a latency under load benchmark YCSB in it's original form suffers from Coordinated Omission[1] and related measurement issue: * Load is controlled by response time * Measurement does not account for missing time * Measurement starts at beginning of request rather than at intended beginning * Measurement is limited in scope as the histogram does not provide data on overflow values To provide a minimal correction patch the following were implemented: 1. Replace internal histogram implementation with HdrHistogram[2]: HdrHistogram offers a dynamic range of measurement at a given precision and will improve the fidelity of reporting. It allows capturing a much wider range of latencies. HdrHistogram also supports compressed loss-less serialization which enable capturing snapshot histograms from which lower resolution histograms can be constructed for plotting latency over time. Snapshot interval histograms are serialized on status reporting which must be enabled using the '-s' option. 2. Track intended operation start and report latencies from that point in time: Assuming the benchmark sets a target schedule of execution in which every operation is supposed to happen at a given time the benchmark should measure the latency between intended start time and operation completion. This required the introduction of a new measurement point and inevitably includes measuring some of the internal preparation steps of the load generator. These overhead should be negligible in the context of a network hop, but could be corrected for by estimating the load-generator overheads (e.g. by measuring a no-op DB or by measuring the setup time for an operation and deducting that from total). This intended measurement point is only used when there is a target load (specified by the -target paramaeter) This branch supports the following new options: * -p measurementtype=[histogram|hdrhistogram|hdrhistogram+histogram|timeseries] (default=histogram) The new measurement types are hdrhistogram and hdrhistogram+histogram. Default is still histogram, which is the old histogram. Ultimately we would remove the old measurement types and use only HdrHistogram but the old measurement is left in there for comparison sake. * -p measurement.interval=[op|intended|both] (default=op) This new option deferentiates between measured intervals and adds the intended interval(as described) above, and the option to record both the op and intended for comparison. * -p hdrhistogram.fileoutput=[true|false] (default=false) This new option will enable periodical writes of the interval histogram into an output file. The path can be set using '-p hdrhistogram.output.path='. Example parameters: -target 1000 -s -p workload=com.yahoo.ycsb.workloads.CoreWorkload -p basicdb.verbose=false -p basicdb.simulatedelay=4 -p measurement.interval=both -p measurementtype=hdrhistogram -p hdrhistogram.fileoutput=true -p maxexecutiontime=60 Further changes made: * -p status.interval= (default=10) Controls the number of seconds between status reports and therefore between HdrHistogram snapshots reported. * -p basicdb.randomizedelay=[true|false] (default=true) Controls weather the delay simulated by the mock DB is uniformly random or not. Further suggestions: 1. Correction load control: currently after a pause the load generator will do operations back to back to catchup, this leads to a flat out throughput mode of testing as opposed to controlled load. 2. Move to async model: Scenarios where Ops have no dependency could delegate the Op execution to a threadpool and thus separate the request rate control from the synchronous execution of Ops. Measurement would start on queuing for execution. 1. https://groups.google.com/forum/#!msg/mechanical-sympathy/icNZJejUHfE/BfDekfBEs_sJ 2. https://github.com/HdrHistogram/HdrHistogram ================================================ FILE: core/pom.xml ================================================ 4.0.0 com.yahoo.ycsb root 0.14.0-SNAPSHOT core Core YCSB jar 1.9.4 org.apache.htrace htrace-core4 4.1.0-incubating org.codehaus.jackson jackson-mapper-asl ${jackson.api.version} org.codehaus.jackson jackson-core-asl ${jackson.api.version} org.testng testng 6.1.1 test org.hdrhistogram HdrHistogram 2.1.4 src/main/resources true ================================================ FILE: core/src/main/java/com/yahoo/ycsb/BasicDB.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.*; import java.util.Map.Entry; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; /** * Basic DB that just prints out the requested operations, instead of doing them against a database. */ public class BasicDB extends DB { public static final String COUNT = "basicdb.count"; public static final String COUNT_DEFAULT = "false"; public static final String VERBOSE = "basicdb.verbose"; public static final String VERBOSE_DEFAULT = "true"; public static final String SIMULATE_DELAY = "basicdb.simulatedelay"; public static final String SIMULATE_DELAY_DEFAULT = "0"; public static final String RANDOMIZE_DELAY = "basicdb.randomizedelay"; public static final String RANDOMIZE_DELAY_DEFAULT = "true"; protected static final Object MUTEX = new Object(); protected static int counter = 0; protected static Map reads; protected static Map scans; protected static Map updates; protected static Map inserts; protected static Map deletes; protected boolean verbose; protected boolean randomizedelay; protected int todelay; protected boolean count; public BasicDB() { todelay = 0; } protected void delay() { if (todelay > 0) { long delayNs; if (randomizedelay) { delayNs = TimeUnit.MILLISECONDS.toNanos(Utils.random().nextInt(todelay)); if (delayNs == 0) { return; } } else { delayNs = TimeUnit.MILLISECONDS.toNanos(todelay); } final long deadline = System.nanoTime() + delayNs; do { LockSupport.parkNanos(deadline - System.nanoTime()); } while (System.nanoTime() < deadline && !Thread.interrupted()); } } /** * Initialize any state for this DB. * Called once per DB instance; there is one DB instance per client thread. */ public void init() { verbose = Boolean.parseBoolean(getProperties().getProperty(VERBOSE, VERBOSE_DEFAULT)); todelay = Integer.parseInt(getProperties().getProperty(SIMULATE_DELAY, SIMULATE_DELAY_DEFAULT)); randomizedelay = Boolean.parseBoolean(getProperties().getProperty(RANDOMIZE_DELAY, RANDOMIZE_DELAY_DEFAULT)); count = Boolean.parseBoolean(getProperties().getProperty(COUNT, COUNT_DEFAULT)); if (verbose) { synchronized (System.out) { System.out.println("***************** properties *****************"); Properties p = getProperties(); if (p != null) { for (Enumeration e = p.propertyNames(); e.hasMoreElements();) { String k = (String) e.nextElement(); System.out.println("\"" + k + "\"=\"" + p.getProperty(k) + "\""); } } System.out.println("**********************************************"); } } synchronized (MUTEX) { if (counter == 0 && count) { reads = new HashMap(); scans = new HashMap(); updates = new HashMap(); inserts = new HashMap(); deletes = new HashMap(); } counter++; } } protected static final ThreadLocal TL_STRING_BUILDER = new ThreadLocal() { @Override protected StringBuilder initialValue() { return new StringBuilder(); } }; protected static StringBuilder getStringBuilder() { StringBuilder sb = TL_STRING_BUILDER.get(); sb.setLength(0); return sb; } /** * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ public Status read(String table, String key, Set fields, Map result) { delay(); if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("READ ").append(table).append(" ").append(key).append(" [ "); if (fields != null) { for (String f : fields) { sb.append(f).append(" "); } } else { sb.append(""); } sb.append("]"); System.out.println(sb); } if (count) { incCounter(reads, hash(table, key, fields)); } return Status.OK; } /** * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored * in a HashMap. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return Zero on success, a non-zero error code on error */ public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { delay(); if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("SCAN ").append(table).append(" ").append(startkey).append(" ").append(recordcount).append(" [ "); if (fields != null) { for (String f : fields) { sb.append(f).append(" "); } } else { sb.append(""); } sb.append("]"); System.out.println(sb); } if (count) { incCounter(scans, hash(table, startkey, fields)); } return Status.OK; } /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key, overwriting any existing values with the same field name. * * @param table The name of the table * @param key The record key of the record to write. * @param values A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error */ public Status update(String table, String key, Map values) { delay(); if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("UPDATE ").append(table).append(" ").append(key).append(" [ "); if (values != null) { for (Map.Entry entry : values.entrySet()) { sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" "); } } sb.append("]"); System.out.println(sb); } if (count) { incCounter(updates, hash(table, key, values)); } return Status.OK; } /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key. * * @param table The name of the table * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error */ public Status insert(String table, String key, Map values) { delay(); if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("INSERT ").append(table).append(" ").append(key).append(" [ "); if (values != null) { for (Map.Entry entry : values.entrySet()) { sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" "); } } sb.append("]"); System.out.println(sb); } if (count) { incCounter(inserts, hash(table, key, values)); } return Status.OK; } /** * Delete a record from the database. * * @param table The name of the table * @param key The record key of the record to delete. * @return Zero on success, a non-zero error code on error */ public Status delete(String table, String key) { delay(); if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("DELETE ").append(table).append(" ").append(key); System.out.println(sb); } if (count) { incCounter(deletes, (table + key).hashCode()); } return Status.OK; } @Override public void cleanup() { synchronized (MUTEX) { int countDown = --counter; if (count && countDown < 1) { // TODO - would be nice to call something like: // Measurements.getMeasurements().oneOffMeasurement("READS", "Uniques", reads.size()); System.out.println("[READS], Uniques, " + reads.size()); System.out.println("[SCANS], Uniques, " + scans.size()); System.out.println("[UPDATES], Uniques, " + updates.size()); System.out.println("[INSERTS], Uniques, " + inserts.size()); System.out.println("[DELETES], Uniques, " + deletes.size()); } } } /** * Increments the count on the hash in the map. * @param map A non-null map to sync and use for incrementing. * @param hash A hash code to increment. */ protected void incCounter(final Map map, final int hash) { synchronized (map) { Integer ctr = map.get(hash); if (ctr == null) { map.put(hash, 1); } else { map.put(hash, ctr + 1); } } } /** * Hashes the table, key and fields, sorting the fields first for a consistent * hash. * Note that this is expensive as we generate a copy of the fields and a string * buffer to hash on. Hashing on the objects is problematic. * @param table The user table. * @param key The key read or scanned. * @param fields The fields read or scanned. * @return The hash code. */ protected int hash(final String table, final String key, final Set fields) { if (fields == null) { return (table + key).hashCode(); } StringBuilder buf = getStringBuilder().append(table).append(key); List sorted = new ArrayList(fields); Collections.sort(sorted); for (final String field : sorted) { buf.append(field); } return buf.toString().hashCode(); } /** * Hashes the table, key and fields, sorting the fields first for a consistent * hash. * Note that this is expensive as we generate a copy of the fields and a string * buffer to hash on. Hashing on the objects is problematic. * @param table The user table. * @param key The key read or scanned. * @param values The values to hash on. * @return The hash code. */ protected int hash(final String table, final String key, final Map values) { if (values == null) { return (table + key).hashCode(); } final TreeMap sorted = new TreeMap(values); StringBuilder buf = getStringBuilder().append(table).append(key); for (final Entry entry : sorted.entrySet()) { entry.getValue().reset(); buf.append(entry.getKey()) .append(entry.getValue().toString()); } return buf.toString().hashCode(); } /** * Short test of BasicDB */ /* public static void main(String[] args) { BasicDB bdb = new BasicDB(); Properties p = new Properties(); p.setProperty("Sky", "Blue"); p.setProperty("Ocean", "Wet"); bdb.setProperties(p); bdb.init(); HashMap fields = new HashMap(); fields.put("A", "X"); fields.put("B", "Y"); bdb.read("table", "key", null, null); bdb.insert("table", "key", fields); fields = new HashMap(); fields.put("C", "Z"); bdb.update("table", "key", fields); bdb.delete("table", "key"); } */ } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/BasicTSDB.java ================================================ /** * Copyright (c) 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import com.yahoo.ycsb.workloads.TimeSeriesWorkload; /** * Basic DB for printing out time series workloads and/or tracking the distribution * of keys and fields. */ public class BasicTSDB extends BasicDB { /** Time series workload specific counters. */ protected static Map timestamps; protected static Map floats; protected static Map integers; private String timestampKey; private String valueKey; private String tagPairDelimiter; private String queryTimeSpanDelimiter; private long lastTimestamp; @Override public void init() { super.init(); synchronized (MUTEX) { if (timestamps == null) { timestamps = new HashMap(); floats = new HashMap(); integers = new HashMap(); } } timestampKey = getProperties().getProperty( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY, TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT); valueKey = getProperties().getProperty( TimeSeriesWorkload.VALUE_KEY_PROPERTY, TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT); tagPairDelimiter = getProperties().getProperty( TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY, TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY_DEFAULT); queryTimeSpanDelimiter = getProperties().getProperty( TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY, TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY_DEFAULT); } public Status read(String table, String key, Set fields, Map result) { delay(); if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("READ ").append(table).append(" ").append(key).append(" [ "); if (fields != null) { for (String f : fields) { sb.append(f).append(" "); } } else { sb.append(""); } sb.append("]"); System.out.println(sb); } if (count) { Set filtered = null; if (fields != null) { filtered = new HashSet(); for (final String field : fields) { if (field.startsWith(timestampKey)) { String[] parts = field.split(tagPairDelimiter); if (parts[1].contains(queryTimeSpanDelimiter)) { parts = parts[1].split(queryTimeSpanDelimiter); lastTimestamp = Long.parseLong(parts[0]); } else { lastTimestamp = Long.parseLong(parts[1]); } synchronized(timestamps) { Integer ctr = timestamps.get(lastTimestamp); if (ctr == null) { timestamps.put(lastTimestamp, 1); } else { timestamps.put(lastTimestamp, ctr + 1); } } } else { filtered.add(field); } } } incCounter(reads, hash(table, key, filtered)); } return Status.OK; } @Override public Status update(String table, String key, Map values) { delay(); boolean isFloat = false; if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("UPDATE ").append(table).append(" ").append(key).append(" [ "); if (values != null) { final TreeMap tree = new TreeMap(values); for (Map.Entry entry : tree.entrySet()) { if (entry.getKey().equals(timestampKey)) { sb.append(entry.getKey()).append("=") .append(Utils.bytesToLong(entry.getValue().toArray())).append(" "); } else if (entry.getKey().equals(valueKey)) { final NumericByteIterator it = (NumericByteIterator) entry.getValue(); isFloat = it.isFloatingPoint(); sb.append(entry.getKey()).append("=") .append(isFloat ? it.getDouble() : it.getLong()).append(" "); } else { sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" "); } } } sb.append("]"); System.out.println(sb); } if (count) { if (!verbose) { isFloat = ((NumericByteIterator) values.get(valueKey)).isFloatingPoint(); } int hash = hash(table, key, values); incCounter(updates, hash); synchronized(timestamps) { Integer ctr = timestamps.get(lastTimestamp); if (ctr == null) { timestamps.put(lastTimestamp, 1); } else { timestamps.put(lastTimestamp, ctr + 1); } } if (isFloat) { incCounter(floats, hash); } else { incCounter(integers, hash); } } return Status.OK; } @Override public Status insert(String table, String key, Map values) { delay(); boolean isFloat = false; if (verbose) { StringBuilder sb = getStringBuilder(); sb.append("INSERT ").append(table).append(" ").append(key).append(" [ "); if (values != null) { final TreeMap tree = new TreeMap(values); for (Map.Entry entry : tree.entrySet()) { if (entry.getKey().equals(timestampKey)) { sb.append(entry.getKey()).append("=") .append(Utils.bytesToLong(entry.getValue().toArray())).append(" "); } else if (entry.getKey().equals(valueKey)) { final NumericByteIterator it = (NumericByteIterator) entry.getValue(); isFloat = it.isFloatingPoint(); sb.append(entry.getKey()).append("=") .append(isFloat ? it.getDouble() : it.getLong()).append(" "); } else { sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" "); } } } sb.append("]"); System.out.println(sb); } if (count) { if (!verbose) { isFloat = ((NumericByteIterator) values.get(valueKey)).isFloatingPoint(); } int hash = hash(table, key, values); incCounter(inserts, hash); synchronized(timestamps) { Integer ctr = timestamps.get(lastTimestamp); if (ctr == null) { timestamps.put(lastTimestamp, 1); } else { timestamps.put(lastTimestamp, ctr + 1); } } if (isFloat) { incCounter(floats, hash); } else { incCounter(integers, hash); } } return Status.OK; } @Override public void cleanup() { super.cleanup(); if (count && counter < 1) { System.out.println("[TIMESTAMPS], Unique, " + timestamps.size()); System.out.println("[FLOATS], Unique series, " + floats.size()); System.out.println("[INTEGERS], Unique series, " + integers.size()); long minTs = Long.MAX_VALUE; long maxTs = Long.MIN_VALUE; for (final long ts : timestamps.keySet()) { if (ts > maxTs) { maxTs = ts; } if (ts < minTs) { minTs = ts; } } System.out.println("[TIMESTAMPS], Min, " + minTs); System.out.println("[TIMESTAMPS], Max, " + maxTs); } } @Override protected int hash(final String table, final String key, final Map values) { final TreeMap sorted = new TreeMap(); for (final Entry entry : values.entrySet()) { if (entry.getKey().equals(valueKey)) { continue; } else if (entry.getKey().equals(timestampKey)) { lastTimestamp = ((NumericByteIterator) entry.getValue()).getLong(); entry.getValue().reset(); continue; } sorted.put(entry.getKey(), entry.getValue()); } // yeah it's ugly but gives us a unique hash without having to add hashers // to all of the ByteIterators. StringBuilder buf = new StringBuilder().append(table).append(key); for (final Entry entry : sorted.entrySet()) { entry.getValue().reset(); buf.append(entry.getKey()) .append(entry.getValue().toString()); } return buf.toString().hashCode(); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/ByteArrayByteIterator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; /** * A ByteIterator that iterates through a byte array. */ public class ByteArrayByteIterator extends ByteIterator { private final int originalOffset; private byte[] str; private int off; private final int len; public ByteArrayByteIterator(byte[] s) { this.str = s; this.off = 0; this.len = s.length; originalOffset = 0; } public ByteArrayByteIterator(byte[] s, int off, int len) { this.str = s; this.off = off; this.len = off + len; originalOffset = off; } @Override public boolean hasNext() { return off < len; } @Override public byte nextByte() { byte ret = str[off]; off++; return ret; } @Override public long bytesLeft() { return len - off; } @Override public void reset() { off = originalOffset; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/ByteIterator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.Charset; import java.util.Iterator; /** * YCSB-specific buffer class. ByteIterators are designed to support * efficient field generation, and to allow backend drivers that can stream * fields (instead of materializing them in RAM) to do so. *

* YCSB originially used String objects to represent field values. This led to * two performance issues. *

* First, it leads to unnecessary conversions between UTF-16 and UTF-8, both * during field generation, and when passing data to byte-based backend * drivers. *

* Second, Java strings are represented internally using UTF-16, and are * built by appending to a growable array type (StringBuilder or * StringBuffer), then calling a toString() method. This leads to a 4x memory * overhead as field values are being built, which prevented YCSB from * driving large object stores. *

* The StringByteIterator class contains a number of convenience methods for * backend drivers that convert between Map<String,String> and * Map<String,ByteBuffer>. * */ public abstract class ByteIterator implements Iterator { @Override public abstract boolean hasNext(); @Override public Byte next() { throw new UnsupportedOperationException(); } public abstract byte nextByte(); /** @return byte offset immediately after the last valid byte */ public int nextBuf(byte[] buf, int bufOff) { int sz = bufOff; while (sz < buf.length && hasNext()) { buf[sz] = nextByte(); sz++; } return sz; } public abstract long bytesLeft(); @Override public void remove() { throw new UnsupportedOperationException(); } /** Resets the iterator so that it can be consumed again. Not all * implementations support this call. * @throws UnsupportedOperationException if the implementation hasn't implemented * the method. */ public void reset() { throw new UnsupportedOperationException(); } /** Consumes remaining contents of this object, and returns them as a string. */ public String toString() { Charset cset = Charset.forName("UTF-8"); CharBuffer cb = cset.decode(ByteBuffer.wrap(this.toArray())); return cb.toString(); } /** Consumes remaining contents of this object, and returns them as a byte array. */ public byte[] toArray() { long left = bytesLeft(); if (left != (int) left) { throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!"); } byte[] ret = new byte[(int) left]; int off = 0; while (off < ret.length) { off = nextBuf(ret, off); } return ret; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/Client.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter; import org.apache.htrace.core.HTraceConfiguration; import org.apache.htrace.core.TraceScope; import org.apache.htrace.core.Tracer; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; /** * A thread to periodically show the status of the experiment to reassure you that progress is being made. */ class StatusThread extends Thread { // Counts down each of the clients completing private final CountDownLatch completeLatch; // Stores the measurements for the run private final Measurements measurements; // Whether or not to track the JVM stats per run private final boolean trackJVMStats; // The clients that are running. private final List clients; private final String label; private final boolean standardstatus; // The interval for reporting status. private long sleeptimeNs; // JVM max/mins private int maxThreads; private int minThreads = Integer.MAX_VALUE; private long maxUsedMem; private long minUsedMem = Long.MAX_VALUE; private double maxLoadAvg; private double minLoadAvg = Double.MAX_VALUE; private long lastGCCount = 0; private long lastGCTime = 0; /** * Creates a new StatusThread without JVM stat tracking. * * @param completeLatch The latch that each client thread will {@link CountDownLatch#countDown()} * as they complete. * @param clients The clients to collect metrics from. * @param label The label for the status. * @param standardstatus If true the status is printed to stdout in addition to stderr. * @param statusIntervalSeconds The number of seconds between status updates. */ public StatusThread(CountDownLatch completeLatch, List clients, String label, boolean standardstatus, int statusIntervalSeconds) { this(completeLatch, clients, label, standardstatus, statusIntervalSeconds, false); } /** * Creates a new StatusThread. * * @param completeLatch The latch that each client thread will {@link CountDownLatch#countDown()} * as they complete. * @param clients The clients to collect metrics from. * @param label The label for the status. * @param standardstatus If true the status is printed to stdout in addition to stderr. * @param statusIntervalSeconds The number of seconds between status updates. * @param trackJVMStats Whether or not to track JVM stats. */ public StatusThread(CountDownLatch completeLatch, List clients, String label, boolean standardstatus, int statusIntervalSeconds, boolean trackJVMStats) { this.completeLatch = completeLatch; this.clients = clients; this.label = label; this.standardstatus = standardstatus; sleeptimeNs = TimeUnit.SECONDS.toNanos(statusIntervalSeconds); measurements = Measurements.getMeasurements(); this.trackJVMStats = trackJVMStats; } /** * Run and periodically report status. */ @Override public void run() { final long startTimeMs = System.currentTimeMillis(); final long startTimeNanos = System.nanoTime(); long deadline = startTimeNanos + sleeptimeNs; long startIntervalMs = startTimeMs; long lastTotalOps = 0; boolean alldone; do { long nowMs = System.currentTimeMillis(); lastTotalOps = computeStats(startTimeMs, startIntervalMs, nowMs, lastTotalOps); if (trackJVMStats) { measureJVM(); } alldone = waitForClientsUntil(deadline); startIntervalMs = nowMs; deadline += sleeptimeNs; } while (!alldone); if (trackJVMStats) { measureJVM(); } // Print the final stats. computeStats(startTimeMs, startIntervalMs, System.currentTimeMillis(), lastTotalOps); } /** * Computes and prints the stats. * * @param startTimeMs The start time of the test. * @param startIntervalMs The start time of this interval. * @param endIntervalMs The end time (now) for the interval. * @param lastTotalOps The last total operations count. * @return The current operation count. */ private long computeStats(final long startTimeMs, long startIntervalMs, long endIntervalMs, long lastTotalOps) { SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS"); long totalops = 0; long todoops = 0; // Calculate the total number of operations completed. for (ClientThread t : clients) { totalops += t.getOpsDone(); todoops += t.getOpsTodo(); } long interval = endIntervalMs - startTimeMs; double throughput = 1000.0 * (((double) totalops) / (double) interval); double curthroughput = 1000.0 * (((double) (totalops - lastTotalOps)) / ((double) (endIntervalMs - startIntervalMs))); long estremaining = (long) Math.ceil(todoops / throughput); DecimalFormat d = new DecimalFormat("#.##"); String labelString = this.label + format.format(new Date()); StringBuilder msg = new StringBuilder(labelString).append(" ").append(interval / 1000).append(" sec: "); msg.append(totalops).append(" operations; "); if (totalops != 0) { msg.append(d.format(curthroughput)).append(" current ops/sec; "); } if (todoops != 0) { msg.append("est completion in ").append(RemainingFormatter.format(estremaining)); } msg.append(Measurements.getMeasurements().getSummary()); System.err.println(msg); if (standardstatus) { System.out.println(msg); } return totalops; } /** * Waits for all of the client to finish or the deadline to expire. * * @param deadline The current deadline. * @return True if all of the clients completed. */ private boolean waitForClientsUntil(long deadline) { boolean alldone = false; long now = System.nanoTime(); while (!alldone && now < deadline) { try { alldone = completeLatch.await(deadline - now, TimeUnit.NANOSECONDS); } catch (InterruptedException ie) { // If we are interrupted the thread is being asked to shutdown. // Return true to indicate that and reset the interrupt state // of the thread. Thread.currentThread().interrupt(); alldone = true; } now = System.nanoTime(); } return alldone; } /** * Executes the JVM measurements. */ private void measureJVM() { final int threads = Utils.getActiveThreadCount(); if (threads < minThreads) { minThreads = threads; } if (threads > maxThreads) { maxThreads = threads; } measurements.measure("THREAD_COUNT", threads); // TODO - once measurements allow for other number types, switch to using // the raw bytes. Otherwise we can track in MB to avoid negative values // when faced with huge heaps. final int usedMem = Utils.getUsedMemoryMegaBytes(); if (usedMem < minUsedMem) { minUsedMem = usedMem; } if (usedMem > maxUsedMem) { maxUsedMem = usedMem; } measurements.measure("USED_MEM_MB", usedMem); // Some JVMs may not implement this feature so if the value is less than // zero, just ommit it. final double systemLoad = Utils.getSystemLoadAverage(); if (systemLoad >= 0) { // TODO - store the double if measurements allows for them measurements.measure("SYS_LOAD_AVG", (int) systemLoad); if (systemLoad > maxLoadAvg) { maxLoadAvg = systemLoad; } if (systemLoad < minLoadAvg) { minLoadAvg = systemLoad; } } final long gcs = Utils.getGCTotalCollectionCount(); measurements.measure("GCS", (int) (gcs - lastGCCount)); final long gcTime = Utils.getGCTotalTime(); measurements.measure("GCS_TIME", (int) (gcTime - lastGCTime)); lastGCCount = gcs; lastGCTime = gcTime; } /** * @return The maximum threads running during the test. */ public int getMaxThreads() { return maxThreads; } /** * @return The minimum threads running during the test. */ public int getMinThreads() { return minThreads; } /** * @return The maximum memory used during the test. */ public long getMaxUsedMem() { return maxUsedMem; } /** * @return The minimum memory used during the test. */ public long getMinUsedMem() { return minUsedMem; } /** * @return The maximum load average during the test. */ public double getMaxLoadAvg() { return maxLoadAvg; } /** * @return The minimum load average during the test. */ public double getMinLoadAvg() { return minLoadAvg; } /** * @return Whether or not the thread is tracking JVM stats. */ public boolean trackJVMStats() { return trackJVMStats; } } /** * Turn seconds remaining into more useful units. * i.e. if there are hours or days worth of seconds, use them. */ final class RemainingFormatter { private RemainingFormatter() { // not used } public static StringBuilder format(long seconds) { StringBuilder time = new StringBuilder(); long days = TimeUnit.SECONDS.toDays(seconds); if (days > 0) { time.append(days).append(days == 1 ? " day " : " days "); seconds -= TimeUnit.DAYS.toSeconds(days); } long hours = TimeUnit.SECONDS.toHours(seconds); if (hours > 0) { time.append(hours).append(hours == 1 ? " hour " : " hours "); seconds -= TimeUnit.HOURS.toSeconds(hours); } /* Only include minute granularity if we're < 1 day. */ if (days < 1) { long minutes = TimeUnit.SECONDS.toMinutes(seconds); if (minutes > 0) { time.append(minutes).append(minutes == 1 ? " minute " : " minutes "); seconds -= TimeUnit.MINUTES.toSeconds(seconds); } } /* Only bother to include seconds if we're < 1 minute */ if (time.length() == 0) { time.append(seconds).append(time.length() == 1 ? " second " : " seconds "); } return time; } } /** * A thread for executing transactions or data inserts to the database. */ class ClientThread implements Runnable { // Counts down each of the clients completing. private final CountDownLatch completeLatch; private static boolean spinSleep; private DB db; private boolean dotransactions; private Workload workload; private int opcount; private double targetOpsPerMs; private int opsdone; private int threadid; private int threadcount; private Object workloadstate; private Properties props; private long targetOpsTickNs; private final Measurements measurements; /** * Constructor. * * @param db the DB implementation to use * @param dotransactions true to do transactions, false to insert data * @param workload the workload to use * @param props the properties defining the experiment * @param opcount the number of operations (transactions or inserts) to do * @param targetperthreadperms target number of operations per thread per ms * @param completeLatch The latch tracking the completion of all clients. */ public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, int opcount, double targetperthreadperms, CountDownLatch completeLatch) { this.db = db; this.dotransactions = dotransactions; this.workload = workload; this.opcount = opcount; opsdone = 0; if (targetperthreadperms > 0) { targetOpsPerMs = targetperthreadperms; targetOpsTickNs = (long) (1000000 / targetOpsPerMs); } this.props = props; measurements = Measurements.getMeasurements(); spinSleep = Boolean.valueOf(this.props.getProperty("spin.sleep", "false")); this.completeLatch = completeLatch; } public void setThreadId(final int threadId) { threadid = threadId; } public void setThreadCount(final int threadCount) { threadcount = threadCount; } public int getOpsDone() { return opsdone; } @Override public void run() { try { db.init(); } catch (DBException e) { e.printStackTrace(); e.printStackTrace(System.out); return; } try { workloadstate = workload.initThread(props, threadid, threadcount); } catch (WorkloadException e) { e.printStackTrace(); e.printStackTrace(System.out); return; } //NOTE: Switching to using nanoTime and parkNanos for time management here such that the measurements // and the client thread have the same view on time. //spread the thread operations out so they don't all hit the DB at the same time // GH issue 4 - throws exception if _target>1 because random.nextInt argument must be >0 // and the sleep() doesn't make sense for granularities < 1 ms anyway if ((targetOpsPerMs > 0) && (targetOpsPerMs <= 1.0)) { long randomMinorDelay = Utils.random().nextInt((int) targetOpsTickNs); sleepUntil(System.nanoTime() + randomMinorDelay); } try { if (dotransactions) { long startTimeNanos = System.nanoTime(); while (((opcount == 0) || (opsdone < opcount)) && !workload.isStopRequested()) { if (!workload.doTransaction(db, workloadstate)) { break; } opsdone++; throttleNanos(startTimeNanos); } } else { long startTimeNanos = System.nanoTime(); while (((opcount == 0) || (opsdone < opcount)) && !workload.isStopRequested()) { if (!workload.doInsert(db, workloadstate)) { break; } opsdone++; throttleNanos(startTimeNanos); } } } catch (Exception e) { e.printStackTrace(); e.printStackTrace(System.out); System.exit(0); } try { measurements.setIntendedStartTimeNs(0); db.cleanup(); } catch (DBException e) { e.printStackTrace(); e.printStackTrace(System.out); } finally { completeLatch.countDown(); } } private static void sleepUntil(long deadline) { while (System.nanoTime() < deadline) { if (!spinSleep) { LockSupport.parkNanos(deadline - System.nanoTime()); } } } private void throttleNanos(long startTimeNanos) { //throttle the operations if (targetOpsPerMs > 0) { // delay until next tick long deadline = startTimeNanos + opsdone * targetOpsTickNs; sleepUntil(deadline); measurements.setIntendedStartTimeNs(deadline); } } /** * The total amount of work this thread is still expected to do. */ int getOpsTodo() { int todo = opcount - opsdone; return todo < 0 ? 0 : todo; } } /** * Main class for executing YCSB. */ public final class Client { private Client() { //not used } public static final String DEFAULT_RECORD_COUNT = "0"; /** * The target number of operations to perform. */ public static final String OPERATION_COUNT_PROPERTY = "operationcount"; /** * The number of records to load into the database initially. */ public static final String RECORD_COUNT_PROPERTY = "recordcount"; /** * The workload class to be loaded. */ public static final String WORKLOAD_PROPERTY = "workload"; /** * The database class to be used. */ public static final String DB_PROPERTY = "db"; /** * The exporter class to be used. The default is * com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter. */ public static final String EXPORTER_PROPERTY = "exporter"; /** * If set to the path of a file, YCSB will write all output to this file * instead of STDOUT. */ public static final String EXPORT_FILE_PROPERTY = "exportfile"; /** * The number of YCSB client threads to run. */ public static final String THREAD_COUNT_PROPERTY = "threadcount"; /** * Indicates how many inserts to do if less than recordcount. * Useful for partitioning the load among multiple servers if the client is the bottleneck. * Additionally workloads should support the "insertstart" property which tells them which record to start at. */ public static final String INSERT_COUNT_PROPERTY = "insertcount"; /** * Target number of operations per second. */ public static final String TARGET_PROPERTY = "target"; /** * The maximum amount of time (in seconds) for which the benchmark will be run. */ public static final String MAX_EXECUTION_TIME = "maxexecutiontime"; /** * Whether or not this is the transaction phase (run) or not (load). */ public static final String DO_TRANSACTIONS_PROPERTY = "dotransactions"; /** * Whether or not to show status during run. */ public static final String STATUS_PROPERTY = "status"; /** * Use label for status (e.g. to label one experiment out of a whole batch). */ public static final String LABEL_PROPERTY = "label"; /** * An optional thread used to track progress and measure JVM stats. */ private static StatusThread statusthread = null; // HTrace integration related constants. /** * All keys for configuring the tracing system start with this prefix. */ private static final String HTRACE_KEY_PREFIX = "htrace."; private static final String CLIENT_WORKLOAD_INIT_SPAN = "Client#workload_init"; private static final String CLIENT_INIT_SPAN = "Client#init"; private static final String CLIENT_WORKLOAD_SPAN = "Client#workload"; private static final String CLIENT_CLEANUP_SPAN = "Client#cleanup"; private static final String CLIENT_EXPORT_MEASUREMENTS_SPAN = "Client#export_measurements"; public static void usageMessage() { System.out.println("Usage: java com.yahoo.ycsb.Client [options]"); System.out.println("Options:"); System.out.println(" -threads n: execute using n threads (default: 1) - can also be specified as the \n" + " \"threadcount\" property using -p"); System.out.println(" -target n: attempt to do n operations per second (default: unlimited) - can also\n" + " be specified as the \"target\" property using -p"); System.out.println(" -load: run the loading phase of the workload"); System.out.println(" -t: run the transactions phase of the workload (default)"); System.out.println(" -db dbname: specify the name of the DB to use (default: com.yahoo.ycsb.BasicDB) - \n" + " can also be specified as the \"db\" property using -p"); System.out.println(" -P propertyfile: load properties from the given file. Multiple files can"); System.out.println(" be specified, and will be processed in the order specified"); System.out.println(" -p name=value: specify a property to be passed to the DB and workloads;"); System.out.println(" multiple properties can be specified, and override any"); System.out.println(" values in the propertyfile"); System.out.println(" -s: show status during run (default: no status)"); System.out.println(" -l label: use label for status (e.g. to label one experiment out of a whole batch)"); System.out.println(""); System.out.println("Required properties:"); System.out.println(" " + WORKLOAD_PROPERTY + ": the name of the workload class to use (e.g. " + "com.yahoo.ycsb.workloads.CoreWorkload)"); System.out.println(""); System.out.println("To run the transaction phase from multiple servers, start a separate client on each."); System.out.println("To run the load phase from multiple servers, start a separate client on each; additionally,"); System.out.println("use the \"insertcount\" and \"insertstart\" properties to divide up the records " + "to be inserted"); } public static boolean checkRequiredProperties(Properties props) { if (props.getProperty(WORKLOAD_PROPERTY) == null) { System.out.println("Missing property: " + WORKLOAD_PROPERTY); return false; } return true; } /** * Exports the measurements to either sysout or a file using the exporter * loaded from conf. * * @throws IOException Either failed to write to output stream or failed to close it. */ private static void exportMeasurements(Properties props, int opcount, long runtime) throws IOException { MeasurementsExporter exporter = null; try { // if no destination file is provided the results will be written to stdout OutputStream out; String exportFile = props.getProperty(EXPORT_FILE_PROPERTY); if (exportFile == null) { out = System.out; } else { out = new FileOutputStream(exportFile); } // if no exporter is provided the default text one will be used String exporterStr = props.getProperty(EXPORTER_PROPERTY, "com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter"); try { exporter = (MeasurementsExporter) Class.forName(exporterStr).getConstructor(OutputStream.class) .newInstance(out); } catch (Exception e) { System.err.println("Could not find exporter " + exporterStr + ", will use default text reporter."); e.printStackTrace(); exporter = new TextMeasurementsExporter(out); } exporter.write("OVERALL", "RunTime(ms)", runtime); double throughput = 1000.0 * (opcount) / (runtime); exporter.write("OVERALL", "Throughput(ops/sec)", throughput); final Map gcs = Utils.getGCStatst(); long totalGCCount = 0; long totalGCTime = 0; for (final Entry entry : gcs.entrySet()) { exporter.write("TOTAL_GCS_" + entry.getKey(), "Count", entry.getValue()[0]); exporter.write("TOTAL_GC_TIME_" + entry.getKey(), "Time(ms)", entry.getValue()[1]); exporter.write("TOTAL_GC_TIME_%_" + entry.getKey(), "Time(%)", ((double) entry.getValue()[1] / runtime) * (double) 100); totalGCCount += entry.getValue()[0]; totalGCTime += entry.getValue()[1]; } exporter.write("TOTAL_GCs", "Count", totalGCCount); exporter.write("TOTAL_GC_TIME", "Time(ms)", totalGCTime); exporter.write("TOTAL_GC_TIME_%", "Time(%)", ((double) totalGCTime / runtime) * (double) 100); if (statusthread != null && statusthread.trackJVMStats()) { exporter.write("MAX_MEM_USED", "MBs", statusthread.getMaxUsedMem()); exporter.write("MIN_MEM_USED", "MBs", statusthread.getMinUsedMem()); exporter.write("MAX_THREADS", "Count", statusthread.getMaxThreads()); exporter.write("MIN_THREADS", "Count", statusthread.getMinThreads()); exporter.write("MAX_SYS_LOAD_AVG", "Load", statusthread.getMaxLoadAvg()); exporter.write("MIN_SYS_LOAD_AVG", "Load", statusthread.getMinLoadAvg()); } Measurements.getMeasurements().exportMeasurements(exporter); } finally { if (exporter != null) { exporter.close(); } } } @SuppressWarnings("unchecked") public static void main(String[] args) { Properties props = parseArguments(args); boolean status = Boolean.valueOf(props.getProperty(STATUS_PROPERTY, String.valueOf(false))); String label = props.getProperty(LABEL_PROPERTY, ""); long maxExecutionTime = Integer.parseInt(props.getProperty(MAX_EXECUTION_TIME, "0")); //get number of threads, target and db int threadcount = Integer.parseInt(props.getProperty(THREAD_COUNT_PROPERTY, "1")); String dbname = props.getProperty(DB_PROPERTY, "com.yahoo.ycsb.BasicDB"); int target = Integer.parseInt(props.getProperty(TARGET_PROPERTY, "0")); //compute the target throughput double targetperthreadperms = -1; if (target > 0) { double targetperthread = ((double) target) / ((double) threadcount); targetperthreadperms = targetperthread / 1000.0; } Thread warningthread = setupWarningThread(); warningthread.start(); Measurements.setProperties(props); Workload workload = getWorkload(props); final Tracer tracer = getTracer(props, workload); initWorkload(props, warningthread, workload, tracer); System.err.println("Starting test."); final CountDownLatch completeLatch = new CountDownLatch(threadcount); final List clients = initDb(dbname, props, threadcount, targetperthreadperms, workload, tracer, completeLatch); if (status) { boolean standardstatus = false; if (props.getProperty(Measurements.MEASUREMENT_TYPE_PROPERTY, "").compareTo("timeseries") == 0) { standardstatus = true; } int statusIntervalSeconds = Integer.parseInt(props.getProperty("status.interval", "10")); boolean trackJVMStats = props.getProperty(Measurements.MEASUREMENT_TRACK_JVM_PROPERTY, Measurements.MEASUREMENT_TRACK_JVM_PROPERTY_DEFAULT).equals("true"); statusthread = new StatusThread(completeLatch, clients, label, standardstatus, statusIntervalSeconds, trackJVMStats); statusthread.start(); } Thread terminator = null; long st; long en; int opsDone; try (final TraceScope span = tracer.newScope(CLIENT_WORKLOAD_SPAN)) { final Map threads = new HashMap<>(threadcount); for (ClientThread client : clients) { threads.put(new Thread(tracer.wrap(client, "ClientThread")), client); } st = System.currentTimeMillis(); for (Thread t : threads.keySet()) { t.start(); } if (maxExecutionTime > 0) { terminator = new TerminatorThread(maxExecutionTime, threads.keySet(), workload); terminator.start(); } opsDone = 0; for (Map.Entry entry : threads.entrySet()) { try { entry.getKey().join(); opsDone += entry.getValue().getOpsDone(); } catch (InterruptedException ignored) { // ignored } } en = System.currentTimeMillis(); } try { try (final TraceScope span = tracer.newScope(CLIENT_CLEANUP_SPAN)) { if (terminator != null && !terminator.isInterrupted()) { terminator.interrupt(); } if (status) { // wake up status thread if it's asleep statusthread.interrupt(); // at this point we assume all the monitored threads are already gone as per above join loop. try { statusthread.join(); } catch (InterruptedException ignored) { // ignored } } workload.cleanup(); } } catch (WorkloadException e) { e.printStackTrace(); e.printStackTrace(System.out); System.exit(0); } try { try (final TraceScope span = tracer.newScope(CLIENT_EXPORT_MEASUREMENTS_SPAN)) { exportMeasurements(props, opsDone, en - st); } } catch (IOException e) { System.err.println("Could not export measurements, error: " + e.getMessage()); e.printStackTrace(); System.exit(-1); } System.exit(0); } private static List initDb(String dbname, Properties props, int threadcount, double targetperthreadperms, Workload workload, Tracer tracer, CountDownLatch completeLatch) { boolean initFailed = false; boolean dotransactions = Boolean.valueOf(props.getProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(true))); final List clients = new ArrayList<>(threadcount); try (final TraceScope span = tracer.newScope(CLIENT_INIT_SPAN)) { int opcount; if (dotransactions) { opcount = Integer.parseInt(props.getProperty(OPERATION_COUNT_PROPERTY, "0")); } else { if (props.containsKey(INSERT_COUNT_PROPERTY)) { opcount = Integer.parseInt(props.getProperty(INSERT_COUNT_PROPERTY, "0")); } else { opcount = Integer.parseInt(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT)); } } for (int threadid = 0; threadid < threadcount; threadid++) { DB db; try { db = DBFactory.newDB(dbname, props, tracer); } catch (UnknownDBException e) { System.out.println("Unknown DB " + dbname); initFailed = true; break; } int threadopcount = opcount / threadcount; // ensure correct number of operations, in case opcount is not a multiple of threadcount if (threadid < opcount % threadcount) { ++threadopcount; } ClientThread t = new ClientThread(db, dotransactions, workload, props, threadopcount, targetperthreadperms, completeLatch); t.setThreadId(threadid); t.setThreadCount(threadcount); clients.add(t); } if (initFailed) { System.err.println("Error initializing datastore bindings."); System.exit(0); } } return clients; } private static Tracer getTracer(Properties props, Workload workload) { return new Tracer.Builder("YCSB " + workload.getClass().getSimpleName()) .conf(getHTraceConfiguration(props)) .build(); } private static void initWorkload(Properties props, Thread warningthread, Workload workload, Tracer tracer) { try { try (final TraceScope span = tracer.newScope(CLIENT_WORKLOAD_INIT_SPAN)) { workload.init(props); warningthread.interrupt(); } } catch (WorkloadException e) { e.printStackTrace(); e.printStackTrace(System.out); System.exit(0); } } private static HTraceConfiguration getHTraceConfiguration(Properties props) { final Map filteredProperties = new HashMap<>(); for (String key : props.stringPropertyNames()) { if (key.startsWith(HTRACE_KEY_PREFIX)) { filteredProperties.put(key.substring(HTRACE_KEY_PREFIX.length()), props.getProperty(key)); } } return HTraceConfiguration.fromMap(filteredProperties); } private static Thread setupWarningThread() { //show a warning message that creating the workload is taking a while //but only do so if it is taking longer than 2 seconds //(showing the message right away if the setup wasn't taking very long was confusing people) return new Thread() { @Override public void run() { try { sleep(2000); } catch (InterruptedException e) { return; } System.err.println(" (might take a few minutes for large data sets)"); } }; } private static Workload getWorkload(Properties props) { ClassLoader classLoader = Client.class.getClassLoader(); try { Properties projectProp = new Properties(); projectProp.load(classLoader.getResourceAsStream("project.properties")); System.err.println("YCSB Client " + projectProp.getProperty("version")); } catch (IOException e) { System.err.println("Unable to retrieve client version."); } System.err.println(); System.err.println("Loading workload..."); try { Class workloadclass = classLoader.loadClass(props.getProperty(WORKLOAD_PROPERTY)); return (Workload) workloadclass.newInstance(); } catch (Exception e) { e.printStackTrace(); e.printStackTrace(System.out); System.exit(0); } return null; } private static Properties parseArguments(String[] args) { Properties props = new Properties(); System.err.print("Command line:"); for (String arg : args) { System.err.print(" " + arg); } Properties fileprops = new Properties(); int argindex = 0; if (args.length == 0) { usageMessage(); System.out.println("At least one argument specifying a workload is required."); System.exit(0); } while (args[argindex].startsWith("-")) { if (args[argindex].compareTo("-threads") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.out.println("Missing argument value for -threads."); System.exit(0); } int tcount = Integer.parseInt(args[argindex]); props.setProperty(THREAD_COUNT_PROPERTY, String.valueOf(tcount)); argindex++; } else if (args[argindex].compareTo("-target") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.out.println("Missing argument value for -target."); System.exit(0); } int ttarget = Integer.parseInt(args[argindex]); props.setProperty(TARGET_PROPERTY, String.valueOf(ttarget)); argindex++; } else if (args[argindex].compareTo("-load") == 0) { props.setProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(false)); argindex++; } else if (args[argindex].compareTo("-t") == 0) { props.setProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(true)); argindex++; } else if (args[argindex].compareTo("-s") == 0) { props.setProperty(STATUS_PROPERTY, String.valueOf(true)); argindex++; } else if (args[argindex].compareTo("-db") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.out.println("Missing argument value for -db."); System.exit(0); } props.setProperty(DB_PROPERTY, args[argindex]); argindex++; } else if (args[argindex].compareTo("-l") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.out.println("Missing argument value for -l."); System.exit(0); } props.setProperty(LABEL_PROPERTY, args[argindex]); argindex++; } else if (args[argindex].compareTo("-P") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.out.println("Missing argument value for -P."); System.exit(0); } String propfile = args[argindex]; argindex++; Properties myfileprops = new Properties(); try { myfileprops.load(new FileInputStream(propfile)); } catch (IOException e) { System.out.println("Unable to open the properties file " + propfile); System.out.println(e.getMessage()); System.exit(0); } //Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5 for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, myfileprops.getProperty(prop)); } } else if (args[argindex].compareTo("-p") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.out.println("Missing argument value for -p"); System.exit(0); } int eq = args[argindex].indexOf('='); if (eq < 0) { usageMessage(); System.out.println("Argument '-p' expected to be in key=value format (e.g., -p operationcount=99999)"); System.exit(0); } String name = args[argindex].substring(0, eq); String value = args[argindex].substring(eq + 1); props.put(name, value); argindex++; } else { usageMessage(); System.out.println("Unknown option " + args[argindex]); System.exit(0); } if (argindex >= args.length) { break; } } if (argindex != args.length) { usageMessage(); if (argindex < args.length) { System.out.println("An argument value without corresponding argument specifier (e.g., -p, -s) was found. " + "We expected an argument specifier and instead found " + args[argindex]); } else { System.out.println("An argument specifier without corresponding value was found at the end of the supplied " + "command line arguments."); } System.exit(0); } //overwrite file properties with properties from the command line //Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5 for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, props.getProperty(prop)); } props = fileprops; if (!checkRequiredProperties(props)) { System.out.println("Failed check required properties."); System.exit(0); } return props; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/CommandLine.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import com.yahoo.ycsb.workloads.CoreWorkload; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.util.*; /** * A simple command line client to a database, using the appropriate com.yahoo.ycsb.DB implementation. */ public final class CommandLine { private CommandLine() { //not used } public static final String DEFAULT_DB = "com.yahoo.ycsb.BasicDB"; public static void usageMessage() { System.out.println("YCSB Command Line Client"); System.out.println("Usage: java com.yahoo.ycsb.CommandLine [options]"); System.out.println("Options:"); System.out.println(" -P filename: Specify a property file"); System.out.println(" -p name=value: Specify a property value"); System.out.println(" -db classname: Use a specified DB class (can also set the \"db\" property)"); System.out.println(" -table tablename: Use the table name instead of the default \"" + CoreWorkload.TABLENAME_PROPERTY_DEFAULT + "\""); System.out.println(); } public static void help() { System.out.println("Commands:"); System.out.println(" read key [field1 field2 ...] - Read a record"); System.out.println(" scan key recordcount [field1 field2 ...] - Scan starting at key"); System.out.println(" insert key name1=value1 [name2=value2 ...] - Insert a new record"); System.out.println(" update key name1=value1 [name2=value2 ...] - Update a record"); System.out.println(" delete key - Delete a record"); System.out.println(" table [tablename] - Get or [set] the name of the table"); System.out.println(" quit - Quit"); } public static void main(String[] args) { Properties props = new Properties(); Properties fileprops = new Properties(); parseArguments(args, props, fileprops); for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, props.getProperty(prop)); } props = fileprops; System.out.println("YCSB Command Line client"); System.out.println("Type \"help\" for command line help"); System.out.println("Start with \"-help\" for usage info"); String table = props.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT); //create a DB String dbname = props.getProperty(Client.DB_PROPERTY, DEFAULT_DB); ClassLoader classLoader = CommandLine.class.getClassLoader(); DB db = null; try { Class dbclass = classLoader.loadClass(dbname); db = (DB) dbclass.newInstance(); } catch (Exception e) { e.printStackTrace(); System.exit(0); } db.setProperties(props); try { db.init(); } catch (DBException e) { e.printStackTrace(); System.exit(0); } System.out.println("Connected."); //main loop BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); for (;;) { //get user input System.out.print("> "); String input = null; try { input = br.readLine(); } catch (IOException e) { e.printStackTrace(); System.exit(1); } if (input.compareTo("") == 0) { continue; } if (input.compareTo("help") == 0) { help(); continue; } if (input.compareTo("quit") == 0) { break; } String[] tokens = input.split(" "); long st = System.currentTimeMillis(); //handle commands if (tokens[0].compareTo("table") == 0) { handleTable(tokens, table); } else if (tokens[0].compareTo("read") == 0) { handleRead(tokens, table, db); } else if (tokens[0].compareTo("scan") == 0) { handleScan(tokens, table, db); } else if (tokens[0].compareTo("update") == 0) { handleUpdate(tokens, table, db); } else if (tokens[0].compareTo("insert") == 0) { handleInsert(tokens, table, db); } else if (tokens[0].compareTo("delete") == 0) { handleDelete(tokens, table, db); } else { System.out.println("Error: unknown command \"" + tokens[0] + "\""); } System.out.println((System.currentTimeMillis() - st) + " ms"); } } private static void parseArguments(String[] args, Properties props, Properties fileprops) { int argindex = 0; while ((argindex < args.length) && (args[argindex].startsWith("-"))) { if ((args[argindex].compareTo("-help") == 0) || (args[argindex].compareTo("--help") == 0) || (args[argindex].compareTo("-?") == 0) || (args[argindex].compareTo("--?") == 0)) { usageMessage(); System.exit(0); } if (args[argindex].compareTo("-db") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } props.setProperty(Client.DB_PROPERTY, args[argindex]); argindex++; } else if (args[argindex].compareTo("-P") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } String propfile = args[argindex]; argindex++; Properties myfileprops = new Properties(); try { myfileprops.load(new FileInputStream(propfile)); } catch (IOException e) { System.out.println(e.getMessage()); System.exit(0); } for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, myfileprops.getProperty(prop)); } } else if (args[argindex].compareTo("-p") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } int eq = args[argindex].indexOf('='); if (eq < 0) { usageMessage(); System.exit(0); } String name = args[argindex].substring(0, eq); String value = args[argindex].substring(eq + 1); props.put(name, value); argindex++; } else if (args[argindex].compareTo("-table") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } props.put(CoreWorkload.TABLENAME_PROPERTY, args[argindex]); argindex++; } else { System.out.println("Unknown option " + args[argindex]); usageMessage(); System.exit(0); } if (argindex >= args.length) { break; } } if (argindex != args.length) { usageMessage(); System.exit(0); } } private static void handleDelete(String[] tokens, String table, DB db) { if (tokens.length != 2) { System.out.println("Error: syntax is \"delete keyname\""); } else { Status ret = db.delete(table, tokens[1]); System.out.println("Return result: " + ret.getName()); } } private static void handleInsert(String[] tokens, String table, DB db) { if (tokens.length < 3) { System.out.println("Error: syntax is \"insert keyname name1=value1 [name2=value2 ...]\""); } else { HashMap values = new HashMap<>(); for (int i = 2; i < tokens.length; i++) { String[] nv = tokens[i].split("="); values.put(nv[0], new StringByteIterator(nv[1])); } Status ret = db.insert(table, tokens[1], values); System.out.println("Result: " + ret.getName()); } } private static void handleUpdate(String[] tokens, String table, DB db) { if (tokens.length < 3) { System.out.println("Error: syntax is \"update keyname name1=value1 [name2=value2 ...]\""); } else { HashMap values = new HashMap<>(); for (int i = 2; i < tokens.length; i++) { String[] nv = tokens[i].split("="); values.put(nv[0], new StringByteIterator(nv[1])); } Status ret = db.update(table, tokens[1], values); System.out.println("Result: " + ret.getName()); } } private static void handleScan(String[] tokens, String table, DB db) { if (tokens.length < 3) { System.out.println("Error: syntax is \"scan keyname scanlength [field1 field2 ...]\""); } else { Set fields = null; if (tokens.length > 3) { fields = new HashSet<>(); fields.addAll(Arrays.asList(tokens).subList(3, tokens.length)); } Vector> results = new Vector<>(); Status ret = db.scan(table, tokens[1], Integer.parseInt(tokens[2]), fields, results); System.out.println("Result: " + ret.getName()); int record = 0; if (results.isEmpty()) { System.out.println("0 records"); } else { System.out.println("--------------------------------"); } for (Map result : results) { System.out.println("Record " + (record++)); for (Map.Entry ent : result.entrySet()) { System.out.println(ent.getKey() + "=" + ent.getValue()); } System.out.println("--------------------------------"); } } } private static void handleRead(String[] tokens, String table, DB db) { if (tokens.length == 1) { System.out.println("Error: syntax is \"read keyname [field1 field2 ...]\""); } else { Set fields = null; if (tokens.length > 2) { fields = new HashSet<>(); fields.addAll(Arrays.asList(tokens).subList(2, tokens.length)); } HashMap result = new HashMap<>(); Status ret = db.read(table, tokens[1], fields, result); System.out.println("Return code: " + ret.getName()); for (Map.Entry ent : result.entrySet()) { System.out.println(ent.getKey() + "=" + ent.getValue()); } } } private static void handleTable(String[] tokens, String table) { if (tokens.length == 1) { System.out.println("Using table \"" + table + "\""); } else if (tokens.length == 2) { table = tokens[1]; System.out.println("Using table \"" + table + "\""); } else { System.out.println("Error: syntax is \"table tablename\""); } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/DB.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * A layer for accessing a database to be benchmarked. Each thread in the client * will be given its own instance of whatever DB class is to be used in the test. * This class should be constructed using a no-argument constructor, so we can * load it dynamically. Any argument-based initialization should be * done by init(). * * Note that YCSB does not make any use of the return codes returned by this class. * Instead, it keeps a count of the return values and presents them to the user. * * The semantics of methods such as insert, update and delete vary from database * to database. In particular, operations may or may not be durable once these * methods commit, and some systems may return 'success' regardless of whether * or not a tuple with a matching key existed before the call. Rather than dictate * the exact semantics of these methods, we recommend you either implement them * to match the database's default semantics, or the semantics of your * target application. For the sake of comparison between experiments we also * recommend you explain the semantics you chose when presenting performance results. */ public abstract class DB { /** * Properties for configuring this DB. */ private Properties properties = new Properties(); /** * Set the properties for this DB. */ public void setProperties(Properties p) { properties = p; } /** * Get the set of properties for this DB. */ public Properties getProperties() { return properties; } /** * Initialize any state for this DB. * Called once per DB instance; there is one DB instance per client thread. */ public void init() throws DBException { } /** * Cleanup any state for this DB. * Called once per DB instance; there is one DB instance per client thread. */ public void cleanup() throws DBException { } /** * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return The result of the operation. */ public abstract Status read(String table, String key, Set fields, Map result); /** * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored * in a HashMap. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return The result of the operation. */ public abstract Status scan(String table, String startkey, int recordcount, Set fields, Vector> result); /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key, overwriting any existing values with the same field name. * * @param table The name of the table * @param key The record key of the record to write. * @param values A HashMap of field/value pairs to update in the record * @return The result of the operation. */ public abstract Status update(String table, String key, Map values); /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key. * * @param table The name of the table * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return The result of the operation. */ public abstract Status insert(String table, String key, Map values); /** * Delete a record from the database. * * @param table The name of the table * @param key The record key of the record to delete. * @return The result of the operation. */ public abstract Status delete(String table, String key); } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/DBException.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; /** * Something bad happened while interacting with the database. */ public class DBException extends Exception { /** * */ private static final long serialVersionUID = 6646883591588721475L; public DBException(String message) { super(message); } public DBException() { super(); } public DBException(String message, Throwable cause) { super(message, cause); } public DBException(Throwable cause) { super(cause); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/DBFactory.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import org.apache.htrace.core.Tracer; import java.util.Properties; /** * Creates a DB layer by dynamically classloading the specified DB class. */ public final class DBFactory { private DBFactory() { // not used } public static DB newDB(String dbname, Properties properties, final Tracer tracer) throws UnknownDBException { ClassLoader classLoader = DBFactory.class.getClassLoader(); DB ret; try { Class dbclass = classLoader.loadClass(dbname); ret = (DB) dbclass.newInstance(); } catch (Exception e) { e.printStackTrace(); return null; } ret.setProperties(properties); return new DBWrapper(ret, tracer); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/DBWrapper.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc., 2016-2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.Map; import com.yahoo.ycsb.measurements.Measurements; import org.apache.htrace.core.TraceScope; import org.apache.htrace.core.Tracer; import java.util.*; /** * Wrapper around a "real" DB that measures latencies and counts return codes. * Also reports latency separately between OK and failed operations. */ public class DBWrapper extends DB { private final DB db; private final Measurements measurements; private final Tracer tracer; private boolean reportLatencyForEachError = false; private Set latencyTrackedErrors = new HashSet(); private static final String REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY = "reportlatencyforeacherror"; private static final String REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY_DEFAULT = "false"; private static final String LATENCY_TRACKED_ERRORS_PROPERTY = "latencytrackederrors"; private final String scopeStringCleanup; private final String scopeStringDelete; private final String scopeStringInit; private final String scopeStringInsert; private final String scopeStringRead; private final String scopeStringScan; private final String scopeStringUpdate; public DBWrapper(final DB db, final Tracer tracer) { this.db = db; measurements = Measurements.getMeasurements(); this.tracer = tracer; final String simple = db.getClass().getSimpleName(); scopeStringCleanup = simple + "#cleanup"; scopeStringDelete = simple + "#delete"; scopeStringInit = simple + "#init"; scopeStringInsert = simple + "#insert"; scopeStringRead = simple + "#read"; scopeStringScan = simple + "#scan"; scopeStringUpdate = simple + "#update"; } /** * Set the properties for this DB. */ public void setProperties(Properties p) { db.setProperties(p); } /** * Get the set of properties for this DB. */ public Properties getProperties() { return db.getProperties(); } /** * Initialize any state for this DB. * Called once per DB instance; there is one DB instance per client thread. */ public void init() throws DBException { try (final TraceScope span = tracer.newScope(scopeStringInit)) { db.init(); this.reportLatencyForEachError = Boolean.parseBoolean(getProperties(). getProperty(REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY, REPORT_LATENCY_FOR_EACH_ERROR_PROPERTY_DEFAULT)); if (!reportLatencyForEachError) { String latencyTrackedErrorsProperty = getProperties().getProperty(LATENCY_TRACKED_ERRORS_PROPERTY, null); if (latencyTrackedErrorsProperty != null) { this.latencyTrackedErrors = new HashSet(Arrays.asList( latencyTrackedErrorsProperty.split(","))); } } System.err.println("DBWrapper: report latency for each error is " + this.reportLatencyForEachError + " and specific error codes to track" + " for latency are: " + this.latencyTrackedErrors.toString()); } } /** * Cleanup any state for this DB. * Called once per DB instance; there is one DB instance per client thread. */ public void cleanup() throws DBException { try (final TraceScope span = tracer.newScope(scopeStringCleanup)) { long ist = measurements.getIntendedtartTimeNs(); long st = System.nanoTime(); db.cleanup(); long en = System.nanoTime(); measure("CLEANUP", Status.OK, ist, st, en); } } /** * Read a record from the database. Each field/value pair from the result * will be stored in a HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return The result of the operation. */ public Status read(String table, String key, Set fields, Map result) { try (final TraceScope span = tracer.newScope(scopeStringRead)) { long ist = measurements.getIntendedtartTimeNs(); long st = System.nanoTime(); Status res = db.read(table, key, fields, result); long en = System.nanoTime(); measure("READ", res, ist, st, en); measurements.reportStatus("READ", res); return res; } } /** * Perform a range scan for a set of records in the database. * Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return The result of the operation. */ public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try (final TraceScope span = tracer.newScope(scopeStringScan)) { long ist = measurements.getIntendedtartTimeNs(); long st = System.nanoTime(); Status res = db.scan(table, startkey, recordcount, fields, result); long en = System.nanoTime(); measure("SCAN", res, ist, st, en); measurements.reportStatus("SCAN", res); return res; } } private void measure(String op, Status result, long intendedStartTimeNanos, long startTimeNanos, long endTimeNanos) { String measurementName = op; if (result == null || !result.isOk()) { if (this.reportLatencyForEachError || this.latencyTrackedErrors.contains(result.getName())) { measurementName = op + "-" + result.getName(); } else { measurementName = op + "-FAILED"; } } measurements.measure(measurementName, (int) ((endTimeNanos - startTimeNanos) / 1000)); measurements.measureIntended(measurementName, (int) ((endTimeNanos - intendedStartTimeNanos) / 1000)); } /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key, overwriting any existing values with the same field name. * * @param table The name of the table * @param key The record key of the record to write. * @param values A HashMap of field/value pairs to update in the record * @return The result of the operation. */ public Status update(String table, String key, Map values) { try (final TraceScope span = tracer.newScope(scopeStringUpdate)) { long ist = measurements.getIntendedtartTimeNs(); long st = System.nanoTime(); Status res = db.update(table, key, values); long en = System.nanoTime(); measure("UPDATE", res, ist, st, en); measurements.reportStatus("UPDATE", res); return res; } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified * record key. * * @param table The name of the table * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return The result of the operation. */ public Status insert(String table, String key, Map values) { try (final TraceScope span = tracer.newScope(scopeStringInsert)) { long ist = measurements.getIntendedtartTimeNs(); long st = System.nanoTime(); Status res = db.insert(table, key, values); long en = System.nanoTime(); measure("INSERT", res, ist, st, en); measurements.reportStatus("INSERT", res); return res; } } /** * Delete a record from the database. * * @param table The name of the table * @param key The record key of the record to delete. * @return The result of the operation. */ public Status delete(String table, String key) { try (final TraceScope span = tracer.newScope(scopeStringDelete)) { long ist = measurements.getIntendedtartTimeNs(); long st = System.nanoTime(); Status res = db.delete(table, key); long en = System.nanoTime(); measure("DELETE", res, ist, st, en); measurements.reportStatus("DELETE", res); return res; } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/GoodBadUglyDB.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.HashMap; import java.util.Map; import java.util.Random; import java.util.Set; import java.util.Vector; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import static java.util.concurrent.TimeUnit.MICROSECONDS; /** * Basic DB that just prints out the requested operations, instead of doing them against a database. */ public class GoodBadUglyDB extends DB { public static final String SIMULATE_DELAY = "gbudb.delays"; public static final String SIMULATE_DELAY_DEFAULT = "200,1000,10000,50000,100000"; private static final ReadWriteLock DB_ACCESS = new ReentrantReadWriteLock(); private long[] delays; public GoodBadUglyDB() { delays = new long[]{200, 1000, 10000, 50000, 200000}; } private void delay() { final Random random = Utils.random(); double p = random.nextDouble(); int mod; if (p < 0.9) { mod = 0; } else if (p < 0.99) { mod = 1; } else if (p < 0.9999) { mod = 2; } else { mod = 3; } // this will make mod 3 pauses global Lock lock = mod == 3 ? DB_ACCESS.writeLock() : DB_ACCESS.readLock(); if (mod == 3) { System.out.println("OUCH"); } lock.lock(); try { final long baseDelayNs = MICROSECONDS.toNanos(delays[mod]); final int delayRangeNs = (int) (MICROSECONDS.toNanos(delays[mod + 1]) - baseDelayNs); final long delayNs = baseDelayNs + random.nextInt(delayRangeNs); final long deadline = System.nanoTime() + delayNs; do { LockSupport.parkNanos(deadline - System.nanoTime()); } while (System.nanoTime() < deadline && !Thread.interrupted()); } finally { lock.unlock(); } } /** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per client thread. */ public void init() { int i = 0; for (String delay : getProperties().getProperty(SIMULATE_DELAY, SIMULATE_DELAY_DEFAULT).split(",")) { delays[i++] = Long.parseLong(delay); } } /** * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ public Status read(String table, String key, Set fields, Map result) { delay(); return Status.OK; } /** * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored * in a HashMap. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return Zero on success, a non-zero error code on error */ public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { delay(); return Status.OK; } /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key, overwriting any existing values with the same field name. * * @param table The name of the table * @param key The record key of the record to write. * @param values A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error */ public Status update(String table, String key, Map values) { delay(); return Status.OK; } /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key. * * @param table The name of the table * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error */ public Status insert(String table, String key, Map values) { delay(); return Status.OK; } /** * Delete a record from the database. * * @param table The name of the table * @param key The record key of the record to delete. * @return Zero on success, a non-zero error code on error */ public Status delete(String table, String key) { delay(); return Status.OK; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/InputStreamByteIterator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.io.IOException; import java.io.InputStream; /** * A ByteIterator that iterates through an inputstream of bytes. */ public class InputStreamByteIterator extends ByteIterator { private long len; private InputStream ins; private long off; private final boolean resetable; public InputStreamByteIterator(InputStream ins, long len) { this.len = len; this.ins = ins; off = 0; resetable = ins.markSupported(); if (resetable) { ins.mark((int) len); } } @Override public boolean hasNext() { return off < len; } @Override public byte nextByte() { int ret; try { ret = ins.read(); } catch (Exception e) { throw new IllegalStateException(e); } if (ret == -1) { throw new IllegalStateException("Past EOF!"); } off++; return (byte) ret; } @Override public long bytesLeft() { return len - off; } @Override public void reset() { if (resetable) { try { ins.reset(); ins.mark((int) len); } catch (IOException e) { throw new IllegalStateException("Failed to reset the input stream", e); } } throw new UnsupportedOperationException(); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/NumericByteIterator.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; /** * A byte iterator that handles encoding and decoding numeric values. * Currently this iterator can handle 64 bit signed values and double precision * floating point values. */ public class NumericByteIterator extends ByteIterator { private final byte[] payload; private final boolean floatingPoint; private int off; public NumericByteIterator(final long value) { floatingPoint = false; payload = Utils.longToBytes(value); off = 0; } public NumericByteIterator(final double value) { floatingPoint = true; payload = Utils.doubleToBytes(value); off = 0; } @Override public boolean hasNext() { return off < payload.length; } @Override public byte nextByte() { return payload[off++]; } @Override public long bytesLeft() { return payload.length - off; } @Override public void reset() { off = 0; } public long getLong() { if (floatingPoint) { throw new IllegalStateException("Byte iterator is of the type double"); } return Utils.bytesToLong(payload); } public double getDouble() { if (!floatingPoint) { throw new IllegalStateException("Byte iterator is of the type long"); } return Utils.bytesToDouble(payload); } public boolean isFloatingPoint() { return floatingPoint; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/RandomByteIterator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; /** * A ByteIterator that generates a random sequence of bytes. */ public class RandomByteIterator extends ByteIterator { private long len; private long off; private int bufOff; private byte[] buf; @Override public boolean hasNext() { return (off + bufOff) < len; } private void fillBytesImpl(byte[] buffer, int base) { int bytes = Utils.random().nextInt(); switch (buffer.length - base) { default: buffer[base + 5] = (byte) (((bytes >> 25) & 95) + ' '); case 5: buffer[base + 4] = (byte) (((bytes >> 20) & 63) + ' '); case 4: buffer[base + 3] = (byte) (((bytes >> 15) & 31) + ' '); case 3: buffer[base + 2] = (byte) (((bytes >> 10) & 95) + ' '); case 2: buffer[base + 1] = (byte) (((bytes >> 5) & 63) + ' '); case 1: buffer[base + 0] = (byte) (((bytes) & 31) + ' '); case 0: break; } } private void fillBytes() { if (bufOff == buf.length) { fillBytesImpl(buf, 0); bufOff = 0; off += buf.length; } } public RandomByteIterator(long len) { this.len = len; this.buf = new byte[6]; this.bufOff = buf.length; fillBytes(); this.off = 0; } public byte nextByte() { fillBytes(); bufOff++; return buf[bufOff - 1]; } @Override public int nextBuf(byte[] buffer, int bufOffset) { int ret; if (len - off < buffer.length - bufOffset) { ret = (int) (len - off); } else { ret = buffer.length - bufOffset; } int i; for (i = 0; i < ret; i += 6) { fillBytesImpl(buffer, i + bufOffset); } off += ret; return ret + bufOffset; } @Override public long bytesLeft() { return len - off - bufOff; } @Override public void reset() { off = 0; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/Status.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; /** * The result of an operation. */ public class Status { private final String name; private final String description; /** * @param name A short name for the status. * @param description A description of the status. */ public Status(String name, String description) { super(); this.name = name; this.description = description; } public String getName() { return name; } public String getDescription() { return description; } @Override public String toString() { return "Status [name=" + name + ", description=" + description + "]"; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((description == null) ? 0 : description.hashCode()); result = prime * result + ((name == null) ? 0 : name.hashCode()); return result; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } Status other = (Status) obj; if (description == null) { if (other.description != null) { return false; } } else if (!description.equals(other.description)) { return false; } if (name == null) { if (other.name != null) { return false; } } else if (!name.equals(other.name)) { return false; } return true; } /** * Is {@code this} a passing state for the operation: {@link Status#OK} or {@link Status#BATCHED_OK}. * @return true if the operation is successful, false otherwise */ public boolean isOk() { return this == OK || this == BATCHED_OK; } public static final Status OK = new Status("OK", "The operation completed successfully."); public static final Status ERROR = new Status("ERROR", "The operation failed."); public static final Status NOT_FOUND = new Status("NOT_FOUND", "The requested record was not found."); public static final Status NOT_IMPLEMENTED = new Status("NOT_IMPLEMENTED", "The operation is not " + "implemented for the current binding."); public static final Status UNEXPECTED_STATE = new Status("UNEXPECTED_STATE", "The operation reported" + " success, but the result was not as expected."); public static final Status BAD_REQUEST = new Status("BAD_REQUEST", "The request was not valid."); public static final Status FORBIDDEN = new Status("FORBIDDEN", "The operation is forbidden."); public static final Status SERVICE_UNAVAILABLE = new Status("SERVICE_UNAVAILABLE", "Dependant " + "service for the current binding is not available."); public static final Status BATCHED_OK = new Status("BATCHED_OK", "The operation has been batched by " + "the binding to be executed later."); } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/StringByteIterator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.HashMap; import java.util.Map; /** * A ByteIterator that iterates through a string. */ public class StringByteIterator extends ByteIterator { private String str; private int off; /** * Put all of the entries of one map into the other, converting * String values into ByteIterators. */ public static void putAllAsByteIterators(Map out, Map in) { for (Map.Entry entry : in.entrySet()) { out.put(entry.getKey(), new StringByteIterator(entry.getValue())); } } /** * Put all of the entries of one map into the other, converting * ByteIterator values into Strings. */ public static void putAllAsStrings(Map out, Map in) { for (Map.Entry entry : in.entrySet()) { out.put(entry.getKey(), entry.getValue().toString()); } } /** * Create a copy of a map, converting the values from Strings to * StringByteIterators. */ public static Map getByteIteratorMap(Map m) { HashMap ret = new HashMap(); for (Map.Entry entry : m.entrySet()) { ret.put(entry.getKey(), new StringByteIterator(entry.getValue())); } return ret; } /** * Create a copy of a map, converting the values from * StringByteIterators to Strings. */ public static Map getStringMap(Map m) { HashMap ret = new HashMap(); for (Map.Entry entry : m.entrySet()) { ret.put(entry.getKey(), entry.getValue().toString()); } return ret; } public StringByteIterator(String s) { this.str = s; this.off = 0; } @Override public boolean hasNext() { return off < str.length(); } @Override public byte nextByte() { byte ret = (byte) str.charAt(off); off++; return ret; } @Override public long bytesLeft() { return str.length() - off; } @Override public void reset() { off = 0; } /** * Specialization of general purpose toString() to avoid unnecessary * copies. *

* Creating a new StringByteIterator, then calling toString() * yields the original String object, and does not perform any copies * or String conversion operations. *

*/ @Override public String toString() { if (off > 0) { return super.toString(); } else { return str; } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/TerminatorThread.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.Collection; /** * A thread that waits for the maximum specified time and then interrupts all the client * threads passed at initialization of this thread. * * The maximum execution time passed is assumed to be in seconds. * */ public class TerminatorThread extends Thread { private final Collection threads; private long maxExecutionTime; private Workload workload; private long waitTimeOutInMS; public TerminatorThread(long maxExecutionTime, Collection threads, Workload workload) { this.maxExecutionTime = maxExecutionTime; this.threads = threads; this.workload = workload; waitTimeOutInMS = 2000; System.err.println("Maximum execution time specified as: " + maxExecutionTime + " secs"); } public void run() { try { Thread.sleep(maxExecutionTime * 1000); } catch (InterruptedException e) { System.err.println("Could not wait until max specified time, TerminatorThread interrupted."); return; } System.err.println("Maximum time elapsed. Requesting stop for the workload."); workload.requestStop(); System.err.println("Stop requested for workload. Now Joining!"); for (Thread t : threads) { while (t.isAlive()) { try { t.join(waitTimeOutInMS); if (t.isAlive()) { System.out.println("Still waiting for thread " + t.getName() + " to complete. " + "Workload status: " + workload.isStopRequested()); } } catch (InterruptedException e) { // Do nothing. Don't know why I was interrupted. } } } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/UnknownDBException.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; /** * Could not create the specified DB. */ public class UnknownDBException extends Exception { /** * */ private static final long serialVersionUID = 459099842269616836L; public UnknownDBException(String message) { super(message); } public UnknownDBException() { super(); } public UnknownDBException(String message, Throwable cause) { super(message, cause); } public UnknownDBException(Throwable cause) { super(cause); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/Utils.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc., 2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; /** * Utility functions. */ public final class Utils { private Utils() { // not used } private static final Random RAND = new Random(); private static final ThreadLocal RNG = new ThreadLocal(); public static Random random() { Random ret = RNG.get(); if (ret == null) { ret = new Random(RAND.nextLong()); RNG.set(ret); } return ret; } /** * Hash an integer value. */ public static long hash(long val) { return fnvhash64(val); } public static final long FNV_OFFSET_BASIS_64 = 0xCBF29CE484222325L; public static final long FNV_PRIME_64 = 1099511628211L; /** * 64 bit FNV hash. Produces more "random" hashes than (say) String.hashCode(). * * @param val The value to hash. * @return The hash value */ public static long fnvhash64(long val) { //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash long hashval = FNV_OFFSET_BASIS_64; for (int i = 0; i < 8; i++) { long octet = val & 0x00ff; val = val >> 8; hashval = hashval ^ octet; hashval = hashval * FNV_PRIME_64; //hashval = hashval ^ octet; } return Math.abs(hashval); } /** * Reads a big-endian 8-byte long from an offset in the given array. * @param bytes The array to read from. * @return A long integer. * @throws IndexOutOfBoundsException if the byte array is too small. * @throws NullPointerException if the byte array is null. */ public static long bytesToLong(final byte[] bytes) { return (bytes[0] & 0xFFL) << 56 | (bytes[1] & 0xFFL) << 48 | (bytes[2] & 0xFFL) << 40 | (bytes[3] & 0xFFL) << 32 | (bytes[4] & 0xFFL) << 24 | (bytes[5] & 0xFFL) << 16 | (bytes[6] & 0xFFL) << 8 | (bytes[7] & 0xFFL) << 0; } /** * Writes a big-endian 8-byte long at an offset in the given array. * @param val The value to encode. * @throws IndexOutOfBoundsException if the byte array is too small. */ public static byte[] longToBytes(final long val) { final byte[] bytes = new byte[8]; bytes[0] = (byte) (val >>> 56); bytes[1] = (byte) (val >>> 48); bytes[2] = (byte) (val >>> 40); bytes[3] = (byte) (val >>> 32); bytes[4] = (byte) (val >>> 24); bytes[5] = (byte) (val >>> 16); bytes[6] = (byte) (val >>> 8); bytes[7] = (byte) (val >>> 0); return bytes; } /** * Parses the byte array into a double. * The byte array must be at least 8 bytes long and have been encoded using * {@link #doubleToBytes}. If the array is longer than 8 bytes, only the * first 8 bytes are parsed. * @param bytes The byte array to parse, at least 8 bytes. * @return A double value read from the byte array. * @throws IllegalArgumentException if the byte array is not 8 bytes wide. */ public static double bytesToDouble(final byte[] bytes) { if (bytes.length < 8) { throw new IllegalArgumentException("Byte array must be 8 bytes wide."); } return Double.longBitsToDouble(bytesToLong(bytes)); } /** * Encodes the double value as an 8 byte array. * @param val The double value to encode. * @return A byte array of length 8. */ public static byte[] doubleToBytes(final double val) { return longToBytes(Double.doubleToRawLongBits(val)); } /** * Measure the estimated active thread count in the current thread group. * Since this calls {@link Thread.activeCount} it should be called from the * main thread or one started by the main thread. Threads included in the * count can be in any state. * For a more accurate count we could use {@link Thread.getAllStackTraces().size()} * but that freezes the JVM and incurs a high overhead. * @return An estimated thread count, good for showing the thread count * over time. */ public static int getActiveThreadCount() { return Thread.activeCount(); } /** @return The currently used memory in bytes */ public static long getUsedMemoryBytes() { final Runtime runtime = Runtime.getRuntime(); return runtime.totalMemory() - runtime.freeMemory(); } /** @return The currently used memory in megabytes. */ public static int getUsedMemoryMegaBytes() { return (int) (getUsedMemoryBytes() / 1024 / 1024); } /** @return The current system load average if supported by the JDK. * If it's not supported, the value will be negative. */ public static double getSystemLoadAverage() { final OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); return osBean.getSystemLoadAverage(); } /** @return The total number of garbage collections executed for all * memory pools. */ public static long getGCTotalCollectionCount() { final List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); long count = 0; for (final GarbageCollectorMXBean bean : gcBeans) { if (bean.getCollectionCount() < 0) { continue; } count += bean.getCollectionCount(); } return count; } /** @return The total time, in milliseconds, spent in GC. */ public static long getGCTotalTime() { final List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); long time = 0; for (final GarbageCollectorMXBean bean : gcBeans) { if (bean.getCollectionTime() < 0) { continue; } time += bean.getCollectionTime(); } return time; } /** * Returns a map of garbage collectors and their stats. * The first object in the array is the total count since JVM start and the * second is the total time (ms) since JVM start. * If a garbage collectors does not support the collector MXBean, then it * will not be represented in the map. * @return A non-null map of garbage collectors and their metrics. The map * may be empty. */ public static Map getGCStatst() { final List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); final Map map = new HashMap(gcBeans.size()); for (final GarbageCollectorMXBean bean : gcBeans) { if (!bean.isValid() || bean.getCollectionCount() < 0 || bean.getCollectionTime() < 0) { continue; } final Long[] measurements = new Long[]{ bean.getCollectionCount(), bean.getCollectionTime() }; map.put(bean.getName().replace(" ", "_"), measurements); } return map; } /** * Simple Fisher-Yates array shuffle to randomize discrete sets. * @param array The array to randomly shuffle. * @return The shuffled array. */ public static T [] shuffleArray(final T[] array) { for (int i = array.length -1; i > 0; i--) { final int idx = RAND.nextInt(i + 1); final T temp = array[idx]; array[idx] = array[i]; array[i] = temp; } return array; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/Workload.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import java.util.concurrent.atomic.AtomicBoolean; import java.util.Properties; /** * One experiment scenario. One object of this type will * be instantiated and shared among all client threads. This class * should be constructed using a no-argument constructor, so we can * load it dynamically. Any argument-based initialization should be * done by init(). * * If you extend this class, you should support the "insertstart" property. This * allows the Client to proceed from multiple clients on different machines, in case * the client is the bottleneck. For example, if we want to load 1 million records from * 2 machines, the first machine should have insertstart=0 and the second insertstart=500000. Additionally, * the "insertcount" property, which is interpreted by Client, can be used to tell each instance of the * client how many inserts to do. In the example above, both clients should have insertcount=500000. */ public abstract class Workload { public static final String INSERT_START_PROPERTY = "insertstart"; public static final String INSERT_COUNT_PROPERTY = "insertcount"; public static final String INSERT_START_PROPERTY_DEFAULT = "0"; private volatile AtomicBoolean stopRequested = new AtomicBoolean(false); /** Operations available for a database. */ public enum Operation { READ, UPDATE, INSERT, SCAN, DELETE } /** * Initialize the scenario. Create any generators and other shared objects here. * Called once, in the main client thread, before any operations are started. */ public void init(Properties p) throws WorkloadException { } /** * Initialize any state for a particular client thread. Since the scenario object * will be shared among all threads, this is the place to create any state that is specific * to one thread. To be clear, this means the returned object should be created anew on each * call to initThread(); do not return the same object multiple times. * The returned object will be passed to invocations of doInsert() and doTransaction() * for this thread. There should be no side effects from this call; all state should be encapsulated * in the returned object. If you have no state to retain for this thread, return null. (But if you have * no state to retain for this thread, probably you don't need to override initThread().) * * @return false if the workload knows it is done for this thread. Client will terminate the thread. * Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read * traces from a file, return true when there are more to do, false when you are done. */ public Object initThread(Properties p, int mythreadid, int threadcount) throws WorkloadException { return null; } /** * Cleanup the scenario. Called once, in the main client thread, after all operations have completed. */ public void cleanup() throws WorkloadException { } /** * Do one insert operation. Because it will be called concurrently from multiple client threads, this * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be * synchronized, since each thread has its own threadstate instance. */ public abstract boolean doInsert(DB db, Object threadstate); /** * Do one transaction operation. Because it will be called concurrently from multiple client threads, this * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be * synchronized, since each thread has its own threadstate instance. * * @return false if the workload knows it is done for this thread. Client will terminate the thread. * Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read * traces from a file, return true when there are more to do, false when you are done. */ public abstract boolean doTransaction(DB db, Object threadstate); /** * Allows scheduling a request to stop the workload. */ public void requestStop() { stopRequested.set(true); } /** * Check the status of the stop request flag. * @return true if stop was requested, false otherwise. */ public boolean isStopRequested() { return stopRequested.get(); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/WorkloadException.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; /** * The workload tried to do something bad. */ public class WorkloadException extends Exception { /** * */ private static final long serialVersionUID = 8844396756042772132L; public WorkloadException(String message) { super(message); } public WorkloadException() { super(); } public WorkloadException(String message, Throwable cause) { super(message, cause); } public WorkloadException(Throwable cause) { super(cause); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/AcknowledgedCounterGenerator.java ================================================ /** * Copyright (c) 2015-2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.concurrent.locks.ReentrantLock; /** * A CounterGenerator that reports generated integers via lastInt() * only after they have been acknowledged. */ public class AcknowledgedCounterGenerator extends CounterGenerator { /** The size of the window of pending id ack's. 2^20 = {@value} */ static final int WINDOW_SIZE = Integer.rotateLeft(1, 20); /** The mask to use to turn an id into a slot in {@link #window}. */ private static final int WINDOW_MASK = WINDOW_SIZE - 1; private final ReentrantLock lock; private final boolean[] window; private volatile long limit; /** * Create a counter that starts at countstart. */ public AcknowledgedCounterGenerator(long countstart) { super(countstart); lock = new ReentrantLock(); window = new boolean[WINDOW_SIZE]; limit = countstart - 1; } /** * In this generator, the highest acknowledged counter value * (as opposed to the highest generated counter value). */ @Override public Long lastValue() { return limit; } /** * Make a generated counter value available via lastInt(). */ public void acknowledge(long value) { final int currentSlot = (int)(value & WINDOW_MASK); if (window[currentSlot]) { throw new RuntimeException("Too many unacknowledged insertion keys."); } window[currentSlot] = true; if (lock.tryLock()) { // move a contiguous sequence from the window // over to the "limit" variable try { // Only loop through the entire window at most once. long beforeFirstSlot = (limit & WINDOW_MASK); long index; for (index = limit + 1; index != beforeFirstSlot; ++index) { int slot = (int)(index & WINDOW_MASK); if (!window[slot]) { break; } window[slot] = false; } limit = index - 1; } finally { lock.unlock(); } } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; /** * A trivial integer generator that always returns the same value. * */ public class ConstantIntegerGenerator extends NumberGenerator { private final int i; /** * @param i The integer that this generator will always return. */ public ConstantIntegerGenerator(int i) { this.i = i; } @Override public Integer nextValue() { return i; } @Override public double mean() { return i; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/CounterGenerator.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc., Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.concurrent.atomic.AtomicLong; /** * Generates a sequence of integers. * (0, 1, ...) */ public class CounterGenerator extends NumberGenerator { private final AtomicLong counter; /** * Create a counter that starts at countstart. */ public CounterGenerator(long countstart) { counter=new AtomicLong(countstart); } @Override public Long nextValue() { return counter.getAndIncrement(); } @Override public Long lastValue() { return counter.get() - 1; } @Override public double mean() { throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!"); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/DiscreteGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import com.yahoo.ycsb.Utils; import java.util.ArrayList; import java.util.Collection; import static java.util.Objects.requireNonNull; /** * Generates a distribution by choosing from a discrete set of values. */ public class DiscreteGenerator extends Generator { private static class Pair { private double weight; private String value; Pair(double weight, String value) { this.weight = weight; this.value = requireNonNull(value); } } private final Collection values = new ArrayList<>(); private String lastvalue; public DiscreteGenerator() { lastvalue = null; } /** * Generate the next string in the distribution. */ @Override public String nextValue() { double sum = 0; for (Pair p : values) { sum += p.weight; } double val = Utils.random().nextDouble(); for (Pair p : values) { double pw = p.weight / sum; if (val < pw) { return p.value; } val -= pw; } throw new AssertionError("oops. should not get here."); } /** * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet * been called, lastString() should return something reasonable. */ @Override public String lastValue() { if (lastvalue == null) { lastvalue = nextValue(); } return lastvalue; } public void addValue(double weight, String value) { values.add(new Pair(weight, value)); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/ExponentialGenerator.java ================================================ /** * Copyright (c) 2011-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import com.yahoo.ycsb.Utils; /** * A generator of an exponential distribution. It produces a sequence * of time intervals according to an exponential * distribution. Smaller intervals are more frequent than larger * ones, and there is no bound on the length of an interval. When you * construct an instance of this class, you specify a parameter gamma, * which corresponds to the rate at which events occur. * Alternatively, 1/gamma is the average length of an interval. */ public class ExponentialGenerator extends NumberGenerator { // What percentage of the readings should be within the most recent exponential.frac portion of the dataset? public static final String EXPONENTIAL_PERCENTILE_PROPERTY = "exponential.percentile"; public static final String EXPONENTIAL_PERCENTILE_DEFAULT = "95"; // What fraction of the dataset should be accessed exponential.percentile of the time? public static final String EXPONENTIAL_FRAC_PROPERTY = "exponential.frac"; public static final String EXPONENTIAL_FRAC_DEFAULT = "0.8571428571"; // 1/7 /** * The exponential constant to use. */ private double gamma; /******************************* Constructors **************************************/ /** * Create an exponential generator with a mean arrival rate of * gamma. (And half life of 1/gamma). */ public ExponentialGenerator(double mean) { gamma = 1.0 / mean; } public ExponentialGenerator(double percentile, double range) { gamma = -Math.log(1.0 - percentile / 100.0) / range; //1.0/mean; } /****************************************************************************************/ /** * Generate the next item as a long. This distribution will be skewed toward lower values; e.g. 0 will * be the most popular, 1 the next most popular, etc. * @return The next item in the sequence. */ @Override public Double nextValue() { return -Math.log(Utils.random().nextDouble()) / gamma; } @Override public double mean() { return 1.0 / gamma; } public static void main(String[] args) { ExponentialGenerator e = new ExponentialGenerator(90, 100); int j = 0; for (int i = 0; i < 1000; i++) { if (e.nextValue() < 100) { j++; } } System.out.println("Got " + j + " hits. Expect 900"); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/FileGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.io.Reader; /** * A generator, whose sequence is the lines of a file. */ public class FileGenerator extends Generator { private final String filename; private String current; private BufferedReader reader; /** * Create a FileGenerator with the given file. * @param filename The file to read lines from. */ public FileGenerator(String filename) { this.filename = filename; reloadFile(); } /** * Return the next string of the sequence, ie the next line of the file. */ @Override public synchronized String nextValue() { try { current = reader.readLine(); return current; } catch (IOException e) { throw new RuntimeException(e); } } /** * Return the previous read line. */ @Override public String lastValue() { return current; } /** * Reopen the file to reuse values. */ public synchronized void reloadFile() { try (Reader r = reader) { System.err.println("Reload " + filename); reader = new BufferedReader(new FileReader(filename)); } catch (IOException e) { throw new RuntimeException(e); } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/Generator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; /** * An expression that generates a sequence of values, following some distribution (Uniform, Zipfian, Sequential, etc.). */ public abstract class Generator { /** * Generate the next value in the distribution. */ public abstract V nextValue(); /** * Return the previous value generated by the distribution; e.g., returned from the last {@link Generator#nextValue()} * call. * Calling {@link #lastValue()} should not advance the distribution or have any side effects. If {@link #nextValue()} * has not yet been called, {@link #lastValue()} should return something reasonable. */ public abstract V lastValue(); public final String nextString() { V ret = nextValue(); return ret == null ? null : ret.toString(); } public final String lastString() { V ret = lastValue(); return ret == null ? null : ret.toString(); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/HistogramGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import com.yahoo.ycsb.Utils; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; /** * Generate integers according to a histogram distribution. The histogram * buckets are of width one, but the values are multiplied by a block size. * Therefore, instead of drawing sizes uniformly at random within each * bucket, we always draw the largest value in the current bucket, so the value * drawn is always a multiple of blockSize. * * The minimum value this distribution returns is blockSize (not zero). * */ public class HistogramGenerator extends NumberGenerator { private final long blockSize; private final long[] buckets; private long area; private long weightedArea = 0; private double meanSize = 0; public HistogramGenerator(String histogramfile) throws IOException { try (BufferedReader in = new BufferedReader(new FileReader(histogramfile))) { String str; String[] line; ArrayList a = new ArrayList<>(); str = in.readLine(); if (str == null) { throw new IOException("Empty input file!\n"); } line = str.split("\t"); if (line[0].compareTo("BlockSize") != 0) { throw new IOException("First line of histogram is not the BlockSize!\n"); } blockSize = Integer.parseInt(line[1]); while ((str = in.readLine()) != null) { // [0] is the bucket, [1] is the value line = str.split("\t"); a.add(Integer.parseInt(line[0]), Integer.parseInt(line[1])); } buckets = new long[a.size()]; for (int i = 0; i < a.size(); i++) { buckets[i] = a.get(i); } } init(); } public HistogramGenerator(long[] buckets, int blockSize) { this.blockSize = blockSize; this.buckets = buckets; init(); } private void init() { for (int i = 0; i < buckets.length; i++) { area += buckets[i]; weightedArea += i * buckets[i]; } // calculate average file size meanSize = ((double) blockSize) * ((double) weightedArea) / (area); } @Override public Long nextValue() { int number = Utils.random().nextInt((int) area); int i; for (i = 0; i < (buckets.length - 1); i++) { number -= buckets[i]; if (number <= 0) { return (i + 1) * blockSize; } } return i * blockSize; } @Override public double mean() { return meanSize; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc. Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import com.yahoo.ycsb.Utils; import java.util.Random; /** * Generate integers resembling a hotspot distribution where x% of operations * access y% of data items. The parameters specify the bounds for the numbers, * the percentage of the of the interval which comprises the hot set and * the percentage of operations that access the hot set. Numbers of the hot set are * always smaller than any number in the cold set. Elements from the hot set and * the cold set are chose using a uniform distribution. * */ public class HotspotIntegerGenerator extends NumberGenerator { private final long lowerBound; private final long upperBound; private final long hotInterval; private final long coldInterval; private final double hotsetFraction; private final double hotOpnFraction; /** * Create a generator for Hotspot distributions. * * @param lowerBound lower bound of the distribution. * @param upperBound upper bound of the distribution. * @param hotsetFraction percentage of data item * @param hotOpnFraction percentage of operations accessing the hot set. */ public HotspotIntegerGenerator(long lowerBound, long upperBound, double hotsetFraction, double hotOpnFraction) { if (hotsetFraction < 0.0 || hotsetFraction > 1.0) { System.err.println("Hotset fraction out of range. Setting to 0.0"); hotsetFraction = 0.0; } if (hotOpnFraction < 0.0 || hotOpnFraction > 1.0) { System.err.println("Hot operation fraction out of range. Setting to 0.0"); hotOpnFraction = 0.0; } if (lowerBound > upperBound) { System.err.println("Upper bound of Hotspot generator smaller than the lower bound. " + "Swapping the values."); long temp = lowerBound; lowerBound = upperBound; upperBound = temp; } this.lowerBound = lowerBound; this.upperBound = upperBound; this.hotsetFraction = hotsetFraction; long interval = upperBound - lowerBound + 1; this.hotInterval = (int) (interval * hotsetFraction); this.coldInterval = interval - hotInterval; this.hotOpnFraction = hotOpnFraction; } @Override public Long nextValue() { long value = 0; Random random = Utils.random(); if (random.nextDouble() < hotOpnFraction) { // Choose a value from the hot set. value = lowerBound + Math.abs(Utils.random().nextLong()) % hotInterval; } else { // Choose a value from the cold set. value = lowerBound + hotInterval + Math.abs(Utils.random().nextLong()) % coldInterval; } setLastValue(value); return value; } /** * @return the lowerBound */ public long getLowerBound() { return lowerBound; } /** * @return the upperBound */ public long getUpperBound() { return upperBound; } /** * @return the hotsetFraction */ public double getHotsetFraction() { return hotsetFraction; } /** * @return the hotOpnFraction */ public double getHotOpnFraction() { return hotOpnFraction; } @Override public double mean() { return hotOpnFraction * (lowerBound + hotInterval / 2.0) + (1 - hotOpnFraction) * (lowerBound + hotInterval + coldInterval / 2.0); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/IncrementingPrintableStringGenerator.java ================================================ /** * Copyright (c) 2016-2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.*; /** * A generator that produces strings of {@link #length} using a set of code points * from {@link #characterSet}. Each time {@link #nextValue()} is executed, the string * is incremented by one character. Eventually the string may rollover to the beginning * and the user may choose to have the generator throw a NoSuchElementException at that * point or continue incrementing. (By default the generator will continue incrementing). *

* For example, if we set a length of 2 characters and the character set includes * [A, B] then the generator output will be: *

    *
  • AA
    • *
  • AB
    • *
  • BA
    • *
  • BB
    • *
  • AA <-- rolled over
    • *
*

* This class includes some default character sets to choose from including ASCII * and plane 0 UTF. */ public class IncrementingPrintableStringGenerator extends Generator { /** Default string length for the generator. */ public static final int DEFAULTSTRINGLENGTH = 8; /** * Set of all character types that include every symbol other than non-printable * control characters. */ public static final Set CHAR_TYPES_ALL_BUT_CONTROL; static { CHAR_TYPES_ALL_BUT_CONTROL = new HashSet(24); // numbers CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.DECIMAL_DIGIT_NUMBER); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.LETTER_NUMBER); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_NUMBER); // letters CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.UPPERCASE_LETTER); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.LOWERCASE_LETTER); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.TITLECASE_LETTER); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_LETTER); // marks CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.COMBINING_SPACING_MARK); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.NON_SPACING_MARK); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.ENCLOSING_MARK); // punctuation CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.CONNECTOR_PUNCTUATION); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.DASH_PUNCTUATION); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.START_PUNCTUATION); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.END_PUNCTUATION); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.INITIAL_QUOTE_PUNCTUATION); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.FINAL_QUOTE_PUNCTUATION); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_PUNCTUATION); // symbols CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.MATH_SYMBOL); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.CURRENCY_SYMBOL); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.MODIFIER_SYMBOL); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.OTHER_SYMBOL); // separators CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.SPACE_SEPARATOR); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.LINE_SEPARATOR); CHAR_TYPES_ALL_BUT_CONTROL.add((int) Character.PARAGRAPH_SEPARATOR); } /** * Set of character types including only decimals, upper and lower case letters. */ public static final Set CHAR_TYPES_BASIC_ALPHA; static { CHAR_TYPES_BASIC_ALPHA = new HashSet(2); CHAR_TYPES_BASIC_ALPHA.add((int) Character.UPPERCASE_LETTER); CHAR_TYPES_BASIC_ALPHA.add((int) Character.LOWERCASE_LETTER); } /** * Set of character types including only decimals, upper and lower case letters. */ public static final Set CHAR_TYPES_BASIC_ALPHANUMERICS; static { CHAR_TYPES_BASIC_ALPHANUMERICS = new HashSet(3); CHAR_TYPES_BASIC_ALPHANUMERICS.add((int) Character.DECIMAL_DIGIT_NUMBER); CHAR_TYPES_BASIC_ALPHANUMERICS.add((int) Character.UPPERCASE_LETTER); CHAR_TYPES_BASIC_ALPHANUMERICS.add((int) Character.LOWERCASE_LETTER); } /** * Set of character types including only decimals, letter numbers, * other numbers, upper, lower, title case as well as letter modifiers * and other letters. */ public static final Set CHAR_TYPE_EXTENDED_ALPHANUMERICS; static { CHAR_TYPE_EXTENDED_ALPHANUMERICS = new HashSet(8); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.DECIMAL_DIGIT_NUMBER); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.LETTER_NUMBER); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.OTHER_NUMBER); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.UPPERCASE_LETTER); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.LOWERCASE_LETTER); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.TITLECASE_LETTER); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.MODIFIER_LETTER); CHAR_TYPE_EXTENDED_ALPHANUMERICS.add((int) Character.OTHER_LETTER); } /** The character set to iterate over. */ private final int[] characterSet; /** An array indices matching a position in the output string. */ private int[] indices; /** The length of the output string in characters. */ private final int length; /** The last value returned by the generator. Should be null if {@link #nextValue()} * has not been called.*/ private String lastValue; /** Whether or not to throw an exception when the string rolls over. */ private boolean throwExceptionOnRollover; /** Whether or not the generator has rolled over. */ private boolean hasRolledOver; /** * Generates strings of 8 characters using only the upper and lower case alphabetical * characters from the ASCII set. */ public IncrementingPrintableStringGenerator() { this(DEFAULTSTRINGLENGTH, printableBasicAlphaASCIISet()); } /** * Generates strings of {@link #length} characters using only the upper and lower * case alphabetical characters from the ASCII set. * @param length The length of string to return from the generator. * @throws IllegalArgumentException if the length is less than one. */ public IncrementingPrintableStringGenerator(final int length) { this(length, printableBasicAlphaASCIISet()); } /** * Generates strings of {@link #length} characters using the code points in * {@link #characterSet}. * @param length The length of string to return from the generator. * @param characterSet A set of code points to choose from. Code points in the * set can be in any order, not necessarily lexical. * @throws IllegalArgumentException if the length is less than one or the character * set has fewer than one code points. */ public IncrementingPrintableStringGenerator(final int length, final int[] characterSet) { if (length < 1) { throw new IllegalArgumentException("Length must be greater than or equal to 1"); } if (characterSet == null || characterSet.length < 1) { throw new IllegalArgumentException("Character set must have at least one character"); } this.length = length; this.characterSet = characterSet; indices = new int[length]; } @Override public String nextValue() { if (hasRolledOver && throwExceptionOnRollover) { throw new NoSuchElementException("The generator has rolled over to the beginning"); } final StringBuilder buffer = new StringBuilder(length); for (int i = 0; i < length; i++) { buffer.append(Character.toChars(characterSet[indices[i]])); } // increment the indices; for (int i = length - 1; i >= 0; --i) { if (indices[i] >= characterSet.length - 1) { indices[i] = 0; if (i == 0 || characterSet.length == 1 && lastValue != null) { hasRolledOver = true; } } else { ++indices[i]; break; } } lastValue = buffer.toString(); return lastValue; } @Override public String lastValue() { return lastValue; } /** @param exceptionOnRollover Whether or not to throw an exception on rollover. */ public void setThrowExceptionOnRollover(final boolean exceptionOnRollover) { this.throwExceptionOnRollover = exceptionOnRollover; } /** @return Whether or not to throw an exception on rollover. */ public boolean getThrowExceptionOnRollover() { return throwExceptionOnRollover; } /** * Returns an array of printable code points with only the upper and lower * case alphabetical characters from the basic ASCII set. * @return An array of code points */ public static int[] printableBasicAlphaASCIISet() { final List validCharacters = generatePrintableCharacterSet(0, 127, null, false, CHAR_TYPES_BASIC_ALPHA); final int[] characterSet = new int[validCharacters.size()]; for (int i = 0; i < validCharacters.size(); i++) { characterSet[i] = validCharacters.get(i); } return characterSet; } /** * Returns an array of printable code points with the upper and lower case * alphabetical characters as well as the numeric values from the basic * ASCII set. * @return An array of code points */ public static int[] printableBasicAlphaNumericASCIISet() { final List validCharacters = generatePrintableCharacterSet(0, 127, null, false, CHAR_TYPES_BASIC_ALPHANUMERICS); final int[] characterSet = new int[validCharacters.size()]; for (int i = 0; i < validCharacters.size(); i++) { characterSet[i] = validCharacters.get(i); } return characterSet; } /** * Returns an array of printable code points with the entire basic ASCII table, * including spaces. Excludes new lines. * @return An array of code points */ public static int[] fullPrintableBasicASCIISet() { final List validCharacters = generatePrintableCharacterSet(32, 127, null, false, null); final int[] characterSet = new int[validCharacters.size()]; for (int i = 0; i < validCharacters.size(); i++) { characterSet[i] = validCharacters.get(i); } return characterSet; } /** * Returns an array of printable code points with the entire basic ASCII table, * including spaces and new lines. * @return An array of code points */ public static int[] fullPrintableBasicASCIISetWithNewlines() { final List validCharacters = new ArrayList(); validCharacters.add(10); // newline validCharacters.addAll(generatePrintableCharacterSet(32, 127, null, false, null)); final int[] characterSet = new int[validCharacters.size()]; for (int i = 0; i < validCharacters.size(); i++) { characterSet[i] = validCharacters.get(i); } return characterSet; } /** * Returns an array of printable code points the first plane of Unicode characters * including only the alpha-numeric values. * @return An array of code points */ public static int[] printableAlphaNumericPlaneZeroSet() { final List validCharacters = generatePrintableCharacterSet(0, 65535, null, false, CHAR_TYPES_BASIC_ALPHANUMERICS); final int[] characterSet = new int[validCharacters.size()]; for (int i = 0; i < validCharacters.size(); i++) { characterSet[i] = validCharacters.get(i); } return characterSet; } /** * Returns an array of printable code points the first plane of Unicode characters * including all printable characters. * @return An array of code points */ public static int[] fullPrintablePlaneZeroSet() { final List validCharacters = generatePrintableCharacterSet(0, 65535, null, false, CHAR_TYPES_ALL_BUT_CONTROL); final int[] characterSet = new int[validCharacters.size()]; for (int i = 0; i < validCharacters.size(); i++) { characterSet[i] = validCharacters.get(i); } return characterSet; } /** * Generates a list of code points based on a range and filters. * These can be used for generating strings with various ASCII and/or * Unicode printable character sets for use with DBs that may have * character limitations. *

* Note that control, surrogate, format, private use and unassigned * code points are skipped. * @param startCodePoint The starting code point, inclusive. * @param lastCodePoint The final code point, inclusive. * @param characterTypesFilter An optional set of allowable character * types. See {@link Character} for types. * @param isFilterAllowableList Determines whether the {@code allowableTypes} * set is inclusive or exclusive. When true, only those code points that * appear in the list will be included in the resulting set. Otherwise * matching code points are excluded. * @param allowableTypes An optional list of code points for inclusion or * exclusion. * @return A list of code points matching the given range and filters. The * list may be empty but is guaranteed not to be null. */ public static List generatePrintableCharacterSet( final int startCodePoint, final int lastCodePoint, final Set characterTypesFilter, final boolean isFilterAllowableList, final Set allowableTypes) { // since we don't know the final size of the allowable character list we // start with a list then we'll flatten it to an array. final List validCharacters = new ArrayList(lastCodePoint); for (int codePoint = startCodePoint; codePoint <= lastCodePoint; ++codePoint) { if (allowableTypes != null && !allowableTypes.contains(Character.getType(codePoint))) { continue; } else { // skip control points, formats, surrogates, etc final int type = Character.getType(codePoint); if (type == Character.CONTROL || type == Character.SURROGATE || type == Character.FORMAT || type == Character.PRIVATE_USE || type == Character.UNASSIGNED) { continue; } } if (characterTypesFilter != null) { // if the filter is enabled then we need to make sure the code point // is in the allowable list if it's a whitelist or that the code point // is NOT in the list if it's a blacklist. if ((isFilterAllowableList && !characterTypesFilter.contains(codePoint)) || (characterTypesFilter.contains(codePoint))) { continue; } } validCharacters.add(codePoint); } return validCharacters; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/NumberGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; /** * A generator that is capable of generating numeric values. * */ public abstract class NumberGenerator extends Generator { private Number lastVal; /** * Set the last value generated. NumberGenerator subclasses must use this call * to properly set the last value, or the {@link #lastValue()} calls won't work. */ protected void setLastValue(Number last) { lastVal = last; } @Override public Number lastValue() { return lastVal; } /** * Return the expected value (mean) of the values this generator will return. */ public abstract double mean(); } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/RandomDiscreteTimestampGenerator.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.concurrent.TimeUnit; import com.yahoo.ycsb.Utils; /** * A generator that picks from a discrete set of offsets from a base Unix Epoch * timestamp that returns timestamps in a random order with the guarantee that * each timestamp is only returned once. *

* TODO - It would be best to implement some kind of psuedo non-repeating random * generator for this as it's likely OK that some small percentage of values are * repeated. For now we just generate all of the offsets in an array, shuffle * it and then iterate over the array. *

* Note that {@link #MAX_INTERVALS} defines a hard limit on the size of the * offset array so that we don't completely blow out the heap. *

* The constructor parameter {@code intervals} determines how many values will be * returned by the generator. For example, if the {@code interval} is 60 and the * {@code timeUnits} are set to {@link TimeUnit#SECONDS} and {@code intervals} * is set to 60, then the consumer can call {@link #nextValue()} 60 times for * timestamps within an hour. */ public class RandomDiscreteTimestampGenerator extends UnixEpochTimestampGenerator { /** A hard limit on the size of the offsets array to a void using too much heap. */ public static final int MAX_INTERVALS = 16777216; /** The total number of intervals for this generator. */ private final int intervals; // can't be primitives due to the generic params on the sort function :( /** The array of generated offsets from the base time. */ private final Integer[] offsets; /** The current index into the offsets array. */ private int offsetIndex; /** * Ctor that uses the current system time as current. * @param interval The interval between timestamps. * @param timeUnits The time units of the returned Unix Epoch timestamp (as well * as the units for the interval). * @param intervals The total number of intervals for the generator. * @throws IllegalArgumentException if the intervals is larger than {@link #MAX_INTERVALS} */ public RandomDiscreteTimestampGenerator(final long interval, final TimeUnit timeUnits, final int intervals) { super(interval, timeUnits); this.intervals = intervals; offsets = new Integer[intervals]; setup(); } /** * Ctor for supplying a starting timestamp. * The interval between timestamps. * @param timeUnits The time units of the returned Unix Epoch timestamp (as well * as the units for the interval). * @param startTimestamp The start timestamp to use. * NOTE that this must match the time units used for the interval. * If the units are in nanoseconds, provide a nanosecond timestamp {@code System.nanoTime()} * or in microseconds, {@code System.nanoTime() / 1000} * or in millis, {@code System.currentTimeMillis()} * @param intervals The total number of intervals for the generator. * @throws IllegalArgumentException if the intervals is larger than {@link #MAX_INTERVALS} */ public RandomDiscreteTimestampGenerator(final long interval, final TimeUnit timeUnits, final long startTimestamp, final int intervals) { super(interval, timeUnits, startTimestamp); this.intervals = intervals; offsets = new Integer[intervals]; setup(); } /** * Generates the offsets and shuffles the array. */ private void setup() { if (intervals > MAX_INTERVALS) { throw new IllegalArgumentException("Too many intervals for the in-memory " + "array. The limit is " + MAX_INTERVALS + "."); } offsetIndex = 0; for (int i = 0; i < intervals; i++) { offsets[i] = i; } Utils.shuffleArray(offsets); } @Override public Long nextValue() { if (offsetIndex >= offsets.length) { throw new IllegalStateException("Reached the end of the random timestamp " + "intervals: " + offsetIndex); } lastTimestamp = currentTimestamp; currentTimestamp = startTimestamp + (offsets[offsetIndex++] * getOffset(1)); return currentTimestamp; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import com.yahoo.ycsb.Utils; /** * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than * others, according to a zipfian distribution. When you construct an instance of this class, you specify the number * of items in the set to draw from, either by specifying an itemcount (so that the sequence is of items from 0 to * itemcount-1) or by specifying a min and a max (so that the sequence is of items from min to max inclusive). After * you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount). *

* Unlike @ZipfianGenerator, this class scatters the "popular" items across the itemspace. Use this, instead of * @ZipfianGenerator, if you don't want the head of the distribution (the popular items) clustered together. */ public class ScrambledZipfianGenerator extends NumberGenerator { public static final double ZETAN = 26.46902820178302; public static final double USED_ZIPFIAN_CONSTANT = 0.99; public static final long ITEM_COUNT = 10000000000L; private ZipfianGenerator gen; private final long min, max, itemcount; /******************************* Constructors **************************************/ /** * Create a zipfian generator for the specified number of items. * * @param items The number of items in the distribution. */ public ScrambledZipfianGenerator(long items) { this(0, items - 1); } /** * Create a zipfian generator for items between min and max. * * @param min The smallest integer to generate in the sequence. * @param max The largest integer to generate in the sequence. */ public ScrambledZipfianGenerator(long min, long max) { this(min, max, ZipfianGenerator.ZIPFIAN_CONSTANT); } /** * Create a zipfian generator for the specified number of items using the specified zipfian constant. * * @param _items The number of items in the distribution. * @param _zipfianconstant The zipfian constant to use. */ /* // not supported, as the value of zeta depends on the zipfian constant, and we have only precomputed zeta for one zipfian constant public ScrambledZipfianGenerator(long _items, double _zipfianconstant) { this(0,_items-1,_zipfianconstant); } */ /** * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. If you * use a zipfian constant other than 0.99, this will take a long time to complete because we need to recompute zeta. * * @param min The smallest integer to generate in the sequence. * @param max The largest integer to generate in the sequence. * @param zipfianconstant The zipfian constant to use. */ public ScrambledZipfianGenerator(long min, long max, double zipfianconstant) { this.min = min; this.max = max; itemcount = this.max - this.min + 1; if (zipfianconstant == USED_ZIPFIAN_CONSTANT) { gen = new ZipfianGenerator(0, ITEM_COUNT, zipfianconstant, ZETAN); } else { gen = new ZipfianGenerator(0, ITEM_COUNT, zipfianconstant); } } /**************************************************************************************************/ /** * Return the next long in the sequence. */ @Override public Long nextValue() { long ret = gen.nextValue(); ret = min + Utils.fnvhash64(ret) % itemcount; setLastValue(ret); return ret; } public static void main(String[] args) { double newzetan = ZipfianGenerator.zetastatic(ITEM_COUNT, ZipfianGenerator.ZIPFIAN_CONSTANT); System.out.println("zetan: " + newzetan); System.exit(0); ScrambledZipfianGenerator gen = new ScrambledZipfianGenerator(10000); for (int i = 0; i < 1000000; i++) { System.out.println("" + gen.nextValue()); } } /** * since the values are scrambled (hopefully uniformly), the mean is simply the middle of the range. */ @Override public double mean() { return ((min) + max) / 2.0; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/SequentialGenerator.java ================================================ /** * Copyright (c) 2016-2017 YCSB Contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.concurrent.atomic.AtomicLong; /** * Generates a sequence of integers 0, 1, ... */ public class SequentialGenerator extends NumberGenerator { private final AtomicLong counter; private long interval; private long countstart; /** * Create a counter that starts at countstart. */ public SequentialGenerator(long countstart, long countend) { counter = new AtomicLong(); setLastValue(counter.get()); this.countstart = countstart; interval = countend - countstart + 1; } /** * If the generator returns numeric (long) values, return the next value as an long. * Default is to return -1, which is appropriate for generators that do not return numeric values. */ public long nextLong() { long ret = countstart + counter.getAndIncrement() % interval; setLastValue(ret); return ret; } @Override public Number nextValue() { long ret = countstart + counter.getAndIncrement() % interval; setLastValue(ret); return ret; } @Override public Number lastValue() { return counter.get() + 1; } @Override public double mean() { throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!"); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/SkewedLatestGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; /** * Generate a popularity distribution of items, skewed to favor recent items significantly more than older items. */ public class SkewedLatestGenerator extends NumberGenerator { private CounterGenerator basis; private final ZipfianGenerator zipfian; public SkewedLatestGenerator(CounterGenerator basis) { this.basis = basis; zipfian = new ZipfianGenerator(this.basis.lastValue()); nextValue(); } /** * Generate the next string in the distribution, skewed Zipfian favoring the items most recently returned by * the basis generator. */ @Override public Long nextValue() { long max = basis.lastValue(); long next = max - zipfian.nextLong(max); setLastValue(next); return next; } public static void main(String[] args) { SkewedLatestGenerator gen = new SkewedLatestGenerator(new CounterGenerator(1000)); for (int i = 0; i < Integer.parseInt(args[0]); i++) { System.out.println(gen.nextString()); } } @Override public double mean() { throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!"); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/UniformGenerator.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc. Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.ArrayList; import java.util.Collection; import java.util.List; /** * An expression that generates a random value in the specified range. */ public class UniformGenerator extends Generator { private final List values; private String laststring; private final UniformLongGenerator gen; /** * Creates a generator that will return strings from the specified set uniformly randomly. */ public UniformGenerator(Collection values) { this.values = new ArrayList<>(values); laststring = null; gen = new UniformLongGenerator(0, values.size() - 1); } /** * Generate the next string in the distribution. */ @Override public String nextValue() { laststring = values.get(gen.nextValue().intValue()); return laststring; } /** * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet * been called, lastString() should return something reasonable. */ @Override public String lastValue() { if (laststring == null) { nextValue(); } return laststring; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/UniformLongGenerator.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc. Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import com.yahoo.ycsb.Utils; /** * Generates longs randomly uniform from an interval. */ public class UniformLongGenerator extends NumberGenerator { private final long lb, ub, interval; /** * Creates a generator that will return longs uniformly randomly from the * interval [lb,ub] inclusive (that is, lb and ub are possible values) * (lb and ub are possible values). * * @param lb the lower bound (inclusive) of generated values * @param ub the upper bound (inclusive) of generated values */ public UniformLongGenerator(long lb, long ub) { this.lb = lb; this.ub = ub; interval = this.ub - this.lb + 1; } @Override public Long nextValue() { long ret = Math.abs(Utils.random().nextLong()) % interval + lb; setLastValue(ret); return ret; } @Override public double mean() { return ((lb + (long) ub)) / 2.0; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/UnixEpochTimestampGenerator.java ================================================ /** * Copyright (c) 2016-2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.concurrent.TimeUnit; /** * A generator that produces Unix epoch timestamps in seconds, milli, micro or * nanoseconds and increments the stamp a given interval each time * {@link #nextValue()} is called. The result is emitted as a long in the same * way calls to {@code System.currentTimeMillis()} and * {@code System.nanoTime()} behave. *

* By default, the current system time of the host is used as the starting * timestamp. Calling {@link #initalizeTimestamp(long)} can adjust the timestamp * back or forward in time. For example, if a workload will generate an hour of * data at 1 minute intervals, then to set the start timestamp an hour in the past * from the current run, use: * 

{@code
 * UnixEpochTimestampGenerator generator = new UnixEpochTimestampGenerator();
 * generator.initalizeTimestamp(-60);
 * }
* A constructor is also present for setting an explicit start time. * Negative intervals are supported as well for iterating back in time. *

* WARNING: This generator is not thread safe and should not called from multiple * threads. */ public class UnixEpochTimestampGenerator extends Generator { /** The base timestamp used as a starting reference. */ protected long startTimestamp; /** The current timestamp that will be incremented. */ protected long currentTimestamp; /** The last used timestamp. Should always be one interval behind current. */ protected long lastTimestamp; /** The interval to increment by. Multiplied by {@link #timeUnits}. */ protected long interval; /** The units of time the interval represents. */ protected TimeUnit timeUnits; /** * Default ctor with the current system time and a 60 second interval. */ public UnixEpochTimestampGenerator() { this(60, TimeUnit.SECONDS); } /** * Ctor that uses the current system time as current. * @param interval The interval for incrementing the timestamp. * @param timeUnits The units of time the increment represents. */ public UnixEpochTimestampGenerator(final long interval, final TimeUnit timeUnits) { this.interval = interval; this.timeUnits = timeUnits; // move the first timestamp by 1 interval so that the first call to nextValue // returns this timestamp initalizeTimestamp(-1); currentTimestamp -= getOffset(1); lastTimestamp = currentTimestamp; } /** * Ctor for supplying a starting timestamp. * @param interval The interval for incrementing the timestamp. * @param timeUnits The units of time the increment represents. * @param startTimestamp The start timestamp to use. * NOTE that this must match the time units used for the interval. * If the units are in nanoseconds, provide a nanosecond timestamp {@code System.nanoTime()} * or in microseconds, {@code System.nanoTime() / 1000} * or in millis, {@code System.currentTimeMillis()} * or seconds and any interval above, {@code System.currentTimeMillis() / 1000} */ public UnixEpochTimestampGenerator(final long interval, final TimeUnit timeUnits, final long startTimestamp) { this.interval = interval; this.timeUnits = timeUnits; // move the first timestamp by 1 interval so that the first call to nextValue // returns this timestamp currentTimestamp = startTimestamp - getOffset(1); this.startTimestamp = currentTimestamp; lastTimestamp = currentTimestamp - getOffset(1); } /** * Sets the starting timestamp to the current system time plus the interval offset. * E.g. to set the time an hour in the past, supply a value of {@code -60}. * @param intervalOffset The interval to increment or decrement by. */ public void initalizeTimestamp(final long intervalOffset) { switch (timeUnits) { case NANOSECONDS: currentTimestamp = System.nanoTime() + getOffset(intervalOffset); break; case MICROSECONDS: currentTimestamp = (System.nanoTime() / 1000) + getOffset(intervalOffset); break; case MILLISECONDS: currentTimestamp = System.currentTimeMillis() + getOffset(intervalOffset); break; case SECONDS: currentTimestamp = (System.currentTimeMillis() / 1000) + getOffset(intervalOffset); break; case MINUTES: currentTimestamp = (System.currentTimeMillis() / 1000) + getOffset(intervalOffset); break; case HOURS: currentTimestamp = (System.currentTimeMillis() / 1000) + getOffset(intervalOffset); break; case DAYS: currentTimestamp = (System.currentTimeMillis() / 1000) + getOffset(intervalOffset); break; default: throw new IllegalArgumentException("Unhandled time unit type: " + timeUnits); } startTimestamp = currentTimestamp; } @Override public Long nextValue() { lastTimestamp = currentTimestamp; currentTimestamp += getOffset(1); return currentTimestamp; } /** * Returns the proper increment offset to use given the interval and timeunits. * @param intervalOffset The amount of offset to multiply by. * @return An offset value to adjust the timestamp by. */ public long getOffset(final long intervalOffset) { switch (timeUnits) { case NANOSECONDS: case MICROSECONDS: case MILLISECONDS: case SECONDS: return intervalOffset * interval; case MINUTES: return intervalOffset * interval * (long) 60; case HOURS: return intervalOffset * interval * (long) (60 * 60); case DAYS: return intervalOffset * interval * (long) (60 * 60 * 24); default: throw new IllegalArgumentException("Unhandled time unit type: " + timeUnits); } } @Override public Long lastValue() { return lastTimestamp; } /** @return The current timestamp as set by the last call to {@link #nextValue()} */ public long currentValue() { return currentTimestamp; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/ZipfianGenerator.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import com.yahoo.ycsb.Utils; /** * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than * others, according to a zipfian distribution. When you construct an instance of this class, you specify the number * of items in the set to draw from, either by specifying an itemcount (so that the sequence is of items from 0 to * itemcount-1) or by specifying a min and a max (so that the sequence is of items from min to max inclusive). After * you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount). * * Note that the popular items will be clustered together, e.g. item 0 is the most popular, item 1 the second most * popular, and so on (or min is the most popular, min+1 the next most popular, etc.) If you don't want this clustering, * and instead want the popular items scattered throughout the item space, then use ScrambledZipfianGenerator instead. * * Be aware: initializing this generator may take a long time if there are lots of items to choose from (e.g. over a * minute for 100 million objects). This is because certain mathematical values need to be computed to properly * generate a zipfian skew, and one of those values (zeta) is a sum sequence from 1 to n, where n is the itemcount. * Note that if you increase the number of items in the set, we can compute a new zeta incrementally, so it should be * fast unless you have added millions of items. However, if you decrease the number of items, we recompute zeta from * scratch, so this can take a long time. * * The algorithm used here is from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994. */ public class ZipfianGenerator extends NumberGenerator { public static final double ZIPFIAN_CONSTANT = 0.99; /** * Number of items. */ private final long items; /** * Min item to generate. */ private final long base; /** * The zipfian constant to use. */ private final double zipfianconstant; /** * Computed parameters for generating the distribution. */ private double alpha, zetan, eta, theta, zeta2theta; /** * The number of items used to compute zetan the last time. */ private long countforzeta; /** * Flag to prevent problems. If you increase the number of items the zipfian generator is allowed to choose from, * this code will incrementally compute a new zeta value for the larger itemcount. However, if you decrease the * number of items, the code computes zeta from scratch; this is expensive for large itemsets. * Usually this is not intentional; e.g. one thread thinks the number of items is 1001 and calls "nextLong()" with * that item count; then another thread who thinks the number of items is 1000 calls nextLong() with itemcount=1000 * triggering the expensive recomputation. (It is expensive for 100 million items, not really for 1000 items.) Why * did the second thread think there were only 1000 items? maybe it read the item count before the first thread * incremented it. So this flag allows you to say if you really do want that recomputation. If true, then the code * will recompute zeta if the itemcount goes down. If false, the code will assume itemcount only goes up, and never * recompute. */ private boolean allowitemcountdecrease = false; /******************************* Constructors **************************************/ /** * Create a zipfian generator for the specified number of items. * @param items The number of items in the distribution. */ public ZipfianGenerator(long items) { this(0, items - 1); } /** * Create a zipfian generator for items between min and max. * @param min The smallest integer to generate in the sequence. * @param max The largest integer to generate in the sequence. */ public ZipfianGenerator(long min, long max) { this(min, max, ZIPFIAN_CONSTANT); } /** * Create a zipfian generator for the specified number of items using the specified zipfian constant. * * @param items The number of items in the distribution. * @param zipfianconstant The zipfian constant to use. */ public ZipfianGenerator(long items, double zipfianconstant) { this(0, items - 1, zipfianconstant); } /** * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. * @param min The smallest integer to generate in the sequence. * @param max The largest integer to generate in the sequence. * @param zipfianconstant The zipfian constant to use. */ public ZipfianGenerator(long min, long max, double zipfianconstant) { this(min, max, zipfianconstant, zetastatic(max - min + 1, zipfianconstant)); } /** * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant, using * the precomputed value of zeta. * * @param min The smallest integer to generate in the sequence. * @param max The largest integer to generate in the sequence. * @param zipfianconstant The zipfian constant to use. * @param zetan The precomputed zeta constant. */ public ZipfianGenerator(long min, long max, double zipfianconstant, double zetan) { items = max - min + 1; base = min; this.zipfianconstant = zipfianconstant; theta = this.zipfianconstant; zeta2theta = zeta(2, theta); alpha = 1.0 / (1.0 - theta); this.zetan = zetan; countforzeta = items; eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / this.zetan); nextValue(); } /**************************************************************************/ /** * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, * using the zipfian constant thetaVal. Remember the value of n, so if we change the itemcount, we can recompute zeta. * * @param n The number of items to compute zeta over. * @param thetaVal The zipfian constant. */ double zeta(long n, double thetaVal) { countforzeta = n; return zetastatic(n, thetaVal); } /** * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, * using the zipfian constant theta. This is a static version of the function which will not remember n. * @param n The number of items to compute zeta over. * @param theta The zipfian constant. */ static double zetastatic(long n, double theta) { return zetastatic(0, n, theta, 0); } /** * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that * has n items now but used to have st items. Use the zipfian constant thetaVal. Remember the new value of * n so that if we change the itemcount, we'll know to recompute zeta. * * @param st The number of items used to compute the last initialsum * @param n The number of items to compute zeta over. * @param thetaVal The zipfian constant. * @param initialsum The value of zeta we are computing incrementally from. */ double zeta(long st, long n, double thetaVal, double initialsum) { countforzeta = n; return zetastatic(st, n, thetaVal, initialsum); } /** * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of * n so that if we change the itemcount, we'll know to recompute zeta. * @param st The number of items used to compute the last initialsum * @param n The number of items to compute zeta over. * @param theta The zipfian constant. * @param initialsum The value of zeta we are computing incrementally from. */ static double zetastatic(long st, long n, double theta, double initialsum) { double sum = initialsum; for (long i = st; i < n; i++) { sum += 1 / (Math.pow(i + 1, theta)); } //System.out.println("countforzeta="+countforzeta); return sum; } /****************************************************************************************/ /** * Generate the next item as a long. * * @param itemcount The number of items in the distribution. * @return The next item in the sequence. */ long nextLong(long itemcount) { //from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994 if (itemcount != countforzeta) { //have to recompute zetan and eta, since they depend on itemcount synchronized (this) { if (itemcount > countforzeta) { //System.err.println("WARNING: Incrementally recomputing Zipfian distribtion. (itemcount="+itemcount+" // countforzeta="+countforzeta+")"); //we have added more items. can compute zetan incrementally, which is cheaper zetan = zeta(countforzeta, itemcount, theta, zetan); eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan); } else if ((itemcount < countforzeta) && (allowitemcountdecrease)) { //have to start over with zetan //note : for large itemsets, this is very slow. so don't do it! //TODO: can also have a negative incremental computation, e.g. if you decrease the number of items, // then just subtract the zeta sequence terms for the items that went away. This would be faster than // recomputing from scratch when the number of items decreases System.err.println("WARNING: Recomputing Zipfian distribtion. This is slow and should be avoided. " + "(itemcount=" + itemcount + " countforzeta=" + countforzeta + ")"); zetan = zeta(itemcount, theta); eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan); } } } double u = Utils.random().nextDouble(); double uz = u * zetan; if (uz < 1.0) { return base; } if (uz < 1.0 + Math.pow(0.5, theta)) { return base + 1; } long ret = base + (long) ((itemcount) * Math.pow(eta * u - eta + 1, alpha)); setLastValue(ret); return ret; } /** * Return the next value, skewed by the Zipfian distribution. The 0th item will be the most popular, followed by * the 1st, followed by the 2nd, etc. (Or, if min != 0, the min-th item is the most popular, the min+1th item the * next most popular, etc.) If you want the popular items scattered throughout the item space, use * ScrambledZipfianGenerator instead. */ @Override public Long nextValue() { return nextLong(items); } public static void main(String[] args) { new ZipfianGenerator(ScrambledZipfianGenerator.ITEM_COUNT); } /** * @todo Implement ZipfianGenerator.mean() */ @Override public double mean() { throw new UnsupportedOperationException("@todo implement ZipfianGenerator.mean()"); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/generator/package-info.java ================================================ /* * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB generator package. */ package com.yahoo.ycsb.generator; ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/Measurements.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import java.io.IOException; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; /** * Collects latency measurements, and reports them when requested. */ public class Measurements { /** * All supported measurement types are defined in this enum. */ public enum MeasurementType { HISTOGRAM, HDRHISTOGRAM, HDRHISTOGRAM_AND_HISTOGRAM, HDRHISTOGRAM_AND_RAW, TIMESERIES, RAW } public static final String MEASUREMENT_TYPE_PROPERTY = "measurementtype"; private static final String MEASUREMENT_TYPE_PROPERTY_DEFAULT = "hdrhistogram"; public static final String MEASUREMENT_INTERVAL = "measurement.interval"; private static final String MEASUREMENT_INTERVAL_DEFAULT = "op"; public static final String MEASUREMENT_TRACK_JVM_PROPERTY = "measurement.trackjvm"; public static final String MEASUREMENT_TRACK_JVM_PROPERTY_DEFAULT = "false"; private static Measurements singleton = null; private static Properties measurementproperties = null; public static void setProperties(Properties props) { measurementproperties = props; } /** * Return the singleton Measurements object. */ public static synchronized Measurements getMeasurements() { if (singleton == null) { singleton = new Measurements(measurementproperties); } return singleton; } private final ConcurrentHashMap opToMesurementMap; private final ConcurrentHashMap opToIntendedMesurementMap; private final MeasurementType measurementType; private final int measurementInterval; private final Properties props; /** * Create a new object with the specified properties. */ public Measurements(Properties props) { opToMesurementMap = new ConcurrentHashMap<>(); opToIntendedMesurementMap = new ConcurrentHashMap<>(); this.props = props; String mTypeString = this.props.getProperty(MEASUREMENT_TYPE_PROPERTY, MEASUREMENT_TYPE_PROPERTY_DEFAULT); switch (mTypeString) { case "histogram": measurementType = MeasurementType.HISTOGRAM; break; case "hdrhistogram": measurementType = MeasurementType.HDRHISTOGRAM; break; case "hdrhistogram+histogram": measurementType = MeasurementType.HDRHISTOGRAM_AND_HISTOGRAM; break; case "hdrhistogram+raw": measurementType = MeasurementType.HDRHISTOGRAM_AND_RAW; break; case "timeseries": measurementType = MeasurementType.TIMESERIES; break; case "raw": measurementType = MeasurementType.RAW; break; default: throw new IllegalArgumentException("unknown " + MEASUREMENT_TYPE_PROPERTY + "=" + mTypeString); } String mIntervalString = this.props.getProperty(MEASUREMENT_INTERVAL, MEASUREMENT_INTERVAL_DEFAULT); switch (mIntervalString) { case "op": measurementInterval = 0; break; case "intended": measurementInterval = 1; break; case "both": measurementInterval = 2; break; default: throw new IllegalArgumentException("unknown " + MEASUREMENT_INTERVAL + "=" + mIntervalString); } } private OneMeasurement constructOneMeasurement(String name) { switch (measurementType) { case HISTOGRAM: return new OneMeasurementHistogram(name, props); case HDRHISTOGRAM: return new OneMeasurementHdrHistogram(name, props); case HDRHISTOGRAM_AND_HISTOGRAM: return new TwoInOneMeasurement(name, new OneMeasurementHdrHistogram("Hdr" + name, props), new OneMeasurementHistogram("Bucket" + name, props)); case HDRHISTOGRAM_AND_RAW: return new TwoInOneMeasurement(name, new OneMeasurementHdrHistogram("Hdr" + name, props), new OneMeasurementRaw("Raw" + name, props)); case TIMESERIES: return new OneMeasurementTimeSeries(name, props); case RAW: return new OneMeasurementRaw(name, props); default: throw new AssertionError("Impossible to be here. Dead code reached. Bugs?"); } } static class StartTimeHolder { protected long time; long startTime() { if (time == 0) { return System.nanoTime(); } else { return time; } } } private final ThreadLocal tlIntendedStartTime = new ThreadLocal() { protected StartTimeHolder initialValue() { return new StartTimeHolder(); } }; public void setIntendedStartTimeNs(long time) { if (measurementInterval == 0) { return; } tlIntendedStartTime.get().time = time; } public long getIntendedtartTimeNs() { if (measurementInterval == 0) { return 0L; } return tlIntendedStartTime.get().startTime(); } /** * Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured * value. */ public void measure(String operation, int latency) { if (measurementInterval == 1) { return; } try { OneMeasurement m = getOpMeasurement(operation); m.measure(latency); } catch (java.lang.ArrayIndexOutOfBoundsException e) { // This seems like a terribly hacky way to cover up for a bug in the measurement code System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing"); e.printStackTrace(); e.printStackTrace(System.out); } } /** * Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured * value. */ public void measureIntended(String operation, int latency) { if (measurementInterval == 0) { return; } try { OneMeasurement m = getOpIntendedMeasurement(operation); m.measure(latency); } catch (java.lang.ArrayIndexOutOfBoundsException e) { // This seems like a terribly hacky way to cover up for a bug in the measurement code System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing"); e.printStackTrace(); e.printStackTrace(System.out); } } private OneMeasurement getOpMeasurement(String operation) { OneMeasurement m = opToMesurementMap.get(operation); if (m == null) { m = constructOneMeasurement(operation); OneMeasurement oldM = opToMesurementMap.putIfAbsent(operation, m); if (oldM != null) { m = oldM; } } return m; } private OneMeasurement getOpIntendedMeasurement(String operation) { OneMeasurement m = opToIntendedMesurementMap.get(operation); if (m == null) { final String name = measurementInterval == 1 ? operation : "Intended-" + operation; m = constructOneMeasurement(name); OneMeasurement oldM = opToIntendedMesurementMap.putIfAbsent(operation, m); if (oldM != null) { m = oldM; } } return m; } /** * Report a return code for a single DB operation. */ public void reportStatus(final String operation, final Status status) { OneMeasurement m = measurementInterval == 1 ? getOpIntendedMeasurement(operation) : getOpMeasurement(operation); m.reportStatus(status); } /** * Export the current measurements to a suitable format. * * @param exporter Exporter representing the type of format to write to. * @throws IOException Thrown if the export failed. */ public void exportMeasurements(MeasurementsExporter exporter) throws IOException { for (OneMeasurement measurement : opToMesurementMap.values()) { measurement.exportMeasurements(exporter); } for (OneMeasurement measurement : opToIntendedMesurementMap.values()) { measurement.exportMeasurements(exporter); } } /** * Return a one line summary of the measurements. */ public synchronized String getSummary() { String ret = ""; for (OneMeasurement m : opToMesurementMap.values()) { ret += m.getSummary() + " "; } for (OneMeasurement m : opToIntendedMesurementMap.values()) { ret += m.getSummary() + " "; } return ret; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurement.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; /** * A single measured metric (such as READ LATENCY). */ public abstract class OneMeasurement { private final String name; private final ConcurrentHashMap returncodes; public String getName() { return name; } /** * @param name measurement name */ public OneMeasurement(String name) { this.name = name; this.returncodes = new ConcurrentHashMap<>(); } public abstract void measure(int latency); public abstract String getSummary(); /** * No need for synchronization, using CHM to deal with that. */ public void reportStatus(Status status) { AtomicInteger counter = returncodes.get(status); if (counter == null) { counter = new AtomicInteger(); AtomicInteger other = returncodes.putIfAbsent(status, counter); if (other != null) { counter = other; } } counter.incrementAndGet(); } /** * Export the current measurements to a suitable format. * * @param exporter Exporter representing the type of format to write to. * @throws IOException Thrown if the export failed. */ public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException; protected final void exportStatusCounts(MeasurementsExporter exporter) throws IOException { for (Map.Entry entry : returncodes.entrySet()) { exporter.write(getName(), "Return=" + entry.getKey().getName(), entry.getValue().get()); } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurementHdrHistogram.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import org.HdrHistogram.Histogram; import org.HdrHistogram.HistogramIterationValue; import org.HdrHistogram.HistogramLogWriter; import org.HdrHistogram.Recorder; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; import java.util.Properties; /** * Take measurements and maintain a HdrHistogram of a given metric, such as READ LATENCY. * */ public class OneMeasurementHdrHistogram extends OneMeasurement { // we need one log per measurement histogram private final PrintStream log; private final HistogramLogWriter histogramLogWriter; private final Recorder histogram; private Histogram totalHistogram; /** * The name of the property for deciding what percentile values to output. */ public static final String PERCENTILES_PROPERTY = "hdrhistogram.percentiles"; /** * The default value for the hdrhistogram.percentiles property. */ public static final String PERCENTILES_PROPERTY_DEFAULT = "95,99"; private final List percentiles; public OneMeasurementHdrHistogram(String name, Properties props) { super(name); percentiles = getPercentileValues(props.getProperty(PERCENTILES_PROPERTY, PERCENTILES_PROPERTY_DEFAULT)); boolean shouldLog = Boolean.parseBoolean(props.getProperty("hdrhistogram.fileoutput", "false")); if (!shouldLog) { log = null; histogramLogWriter = null; } else { try { final String hdrOutputFilename = props.getProperty("hdrhistogram.output.path", "") + name + ".hdr"; log = new PrintStream(new FileOutputStream(hdrOutputFilename), false); } catch (FileNotFoundException e) { throw new RuntimeException("Failed to open hdr histogram output file", e); } histogramLogWriter = new HistogramLogWriter(log); histogramLogWriter.outputComment("[Logging for: " + name + "]"); histogramLogWriter.outputLogFormatVersion(); long now = System.currentTimeMillis(); histogramLogWriter.outputStartTime(now); histogramLogWriter.setBaseTime(now); histogramLogWriter.outputLegend(); } histogram = new Recorder(3); } /** * It appears latency is reported in micros. * Using {@link Recorder} to support concurrent updates to histogram. */ public void measure(int latencyInMicros) { histogram.recordValue(latencyInMicros); } /** * This is called from a main thread, on orderly termination. */ @Override public void exportMeasurements(MeasurementsExporter exporter) throws IOException { // accumulate the last interval which was not caught by status thread Histogram intervalHistogram = getIntervalHistogramAndAccumulate(); if (histogramLogWriter != null) { histogramLogWriter.outputIntervalHistogram(intervalHistogram); // we can close now log.close(); } exporter.write(getName(), "Operations", totalHistogram.getTotalCount()); exporter.write(getName(), "AverageLatency(us)", totalHistogram.getMean()); exporter.write(getName(), "MinLatency(us)", totalHistogram.getMinValue()); exporter.write(getName(), "MaxLatency(us)", totalHistogram.getMaxValue()); for (Double percentile : percentiles) { exporter.write(getName(), ordinal(percentile) + "PercentileLatency(us)", totalHistogram.getValueAtPercentile(percentile)); } exportStatusCounts(exporter); // also export totalHistogram for (HistogramIterationValue v : totalHistogram.recordedValues()) { int value; if (v.getValueIteratedTo() > (long)Integer.MAX_VALUE) { value = Integer.MAX_VALUE; } else { value = (int)v.getValueIteratedTo(); } exporter.write(getName(), Integer.toString(value), (double)v.getCountAtValueIteratedTo()); } } /** * This is called periodically from the StatusThread. There's a single * StatusThread per Client process. We optionally serialize the interval to * log on this opportunity. * * @see com.yahoo.ycsb.measurements.OneMeasurement#getSummary() */ @Override public String getSummary() { Histogram intervalHistogram = getIntervalHistogramAndAccumulate(); // we use the summary interval as the histogram file interval. if (histogramLogWriter != null) { histogramLogWriter.outputIntervalHistogram(intervalHistogram); } DecimalFormat d = new DecimalFormat("#.##"); return "[" + getName() + ": Count=" + intervalHistogram.getTotalCount() + ", Max=" + intervalHistogram.getMaxValue() + ", Min=" + intervalHistogram.getMinValue() + ", Avg=" + d.format(intervalHistogram.getMean()) + ", 90=" + d.format(intervalHistogram.getValueAtPercentile(90)) + ", 99=" + d.format(intervalHistogram.getValueAtPercentile(99)) + ", 99.9=" + d.format(intervalHistogram.getValueAtPercentile(99.9)) + ", 99.99=" + d.format(intervalHistogram.getValueAtPercentile(99.99)) + "]"; } private Histogram getIntervalHistogramAndAccumulate() { Histogram intervalHistogram = histogram.getIntervalHistogram(); // add this to the total time histogram. if (totalHistogram == null) { totalHistogram = intervalHistogram; } else { totalHistogram.add(intervalHistogram); } return intervalHistogram; } /** * Helper method to parse the given percentile value string. * * @param percentileString - comma delimited string of Integer values * @return An Integer List of percentile values */ private List getPercentileValues(String percentileString) { List percentileValues = new ArrayList<>(); try { for (String rawPercentile : percentileString.split(",")) { percentileValues.add(Double.parseDouble(rawPercentile)); } } catch (Exception e) { // If the given hdrhistogram.percentiles value is unreadable for whatever reason, // then calculate and return the default set. System.err.println("[WARN] Couldn't read " + PERCENTILES_PROPERTY + " value: '" + percentileString + "', the default of '" + PERCENTILES_PROPERTY_DEFAULT + "' will be used."); e.printStackTrace(); return getPercentileValues(PERCENTILES_PROPERTY_DEFAULT); } return percentileValues; } /** * Helper method to find the ordinal of any number. eg 1 -> 1st * @param i number * @return ordinal string */ private String ordinal(Double i) { String[] suffixes = new String[]{"th", "st", "nd", "rd", "th", "th", "th", "th", "th", "th"}; Integer j = i.intValue(); if (i % 1 == 0) { switch (j % 100) { case 11: case 12: case 13: return j + "th"; default: return j + suffixes[j % 10]; } } else { return i.toString(); } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import java.io.IOException; import java.text.DecimalFormat; import java.util.Properties; /** * Take measurements and maintain a histogram of a given metric, such as READ LATENCY. * */ public class OneMeasurementHistogram extends OneMeasurement { public static final String BUCKETS = "histogram.buckets"; public static final String BUCKETS_DEFAULT = "1000"; /** * Specify the range of latencies to track in the histogram. */ private final int buckets; /** * Groups operations in discrete blocks of 1ms width. */ private long[] histogram; /** * Counts all operations outside the histogram's range. */ private long histogramoverflow; /** * The total number of reported operations. */ private long operations; /** * The sum of each latency measurement over all operations. * Calculated in ms. */ private long totallatency; /** * The sum of each latency measurement squared over all operations. * Used to calculate variance of latency. * Calculated in ms. */ private double totalsquaredlatency; //keep a windowed version of these stats for printing status private long windowoperations; private long windowtotallatency; private int min; private int max; public OneMeasurementHistogram(String name, Properties props) { super(name); buckets = Integer.parseInt(props.getProperty(BUCKETS, BUCKETS_DEFAULT)); histogram = new long[buckets]; histogramoverflow = 0; operations = 0; totallatency = 0; totalsquaredlatency = 0; windowoperations = 0; windowtotallatency = 0; min = -1; max = -1; } /* (non-Javadoc) * @see com.yahoo.ycsb.OneMeasurement#measure(int) */ public synchronized void measure(int latency) { //latency reported in us and collected in bucket by ms. if (latency / 1000 >= buckets) { histogramoverflow++; } else { histogram[latency / 1000]++; } operations++; totallatency += latency; totalsquaredlatency += ((double) latency) * ((double) latency); windowoperations++; windowtotallatency += latency; if ((min < 0) || (latency < min)) { min = latency; } if ((max < 0) || (latency > max)) { max = latency; } } @Override public void exportMeasurements(MeasurementsExporter exporter) throws IOException { double mean = totallatency / ((double) operations); double variance = totalsquaredlatency / ((double) operations) - (mean * mean); exporter.write(getName(), "Operations", operations); exporter.write(getName(), "AverageLatency(us)", mean); exporter.write(getName(), "LatencyVariance(us)", variance); exporter.write(getName(), "MinLatency(us)", min); exporter.write(getName(), "MaxLatency(us)", max); long opcounter=0; boolean done95th = false; for (int i = 0; i < buckets; i++) { opcounter += histogram[i]; if ((!done95th) && (((double) opcounter) / ((double) operations) >= 0.95)) { exporter.write(getName(), "95thPercentileLatency(us)", i * 1000); done95th = true; } if (((double) opcounter) / ((double) operations) >= 0.99) { exporter.write(getName(), "99thPercentileLatency(us)", i * 1000); break; } } exportStatusCounts(exporter); for (int i = 0; i < buckets; i++) { exporter.write(getName(), Integer.toString(i), histogram[i]); } exporter.write(getName(), ">" + buckets, histogramoverflow); } @Override public String getSummary() { if (windowoperations == 0) { return ""; } DecimalFormat d = new DecimalFormat("#.##"); double report = ((double) windowtotallatency) / ((double) windowoperations); windowtotallatency = 0; windowoperations = 0; return "[" + getName() + " AverageLatency(us)=" + d.format(report) + "]"; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurementRaw.java ================================================ /** * Copyright (c) 2015-2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.Properties; /** * Record a series of measurements as raw data points without down sampling, * optionally write to an output file when configured. * */ public class OneMeasurementRaw extends OneMeasurement { /** * One raw data point, two fields: timestamp (ms) when the datapoint is * inserted, and the value. */ class RawDataPoint { private final long timestamp; private final int value; public RawDataPoint(int value) { this.timestamp = System.currentTimeMillis(); this.value = value; } public long timeStamp() { return timestamp; } public int value() { return value; } } class RawDataPointComparator implements Comparator { @Override public int compare(RawDataPoint p1, RawDataPoint p2) { if (p1.value() < p2.value()) { return -1; } else if (p1.value() == p2.value()) { return 0; } else { return 1; } } } /** * Optionally, user can configure an output file to save the raw data points. * Default is none, raw results will be written to stdout. * */ public static final String OUTPUT_FILE_PATH = "measurement.raw.output_file"; public static final String OUTPUT_FILE_PATH_DEFAULT = ""; /** * Optionally, user can request to not output summary stats. This is useful * if the user chains the raw measurement type behind the HdrHistogram type * which already outputs summary stats. But even in that case, the user may * still want this class to compute summary stats for them, especially if * they want accurate computation of percentiles (because percentils computed * by histogram classes are still approximations). */ public static final String NO_SUMMARY_STATS = "measurement.raw.no_summary"; public static final String NO_SUMMARY_STATS_DEFAULT = "false"; private final PrintStream outputStream; private boolean noSummaryStats = false; private LinkedList measurements; private long totalLatency = 0; // A window of stats to print summary for at the next getSummary() call. // It's supposed to be a one line summary, so we will just print count and // average. private int windowOperations = 0; private long windowTotalLatency = 0; public OneMeasurementRaw(String name, Properties props) { super(name); String outputFilePath = props.getProperty(OUTPUT_FILE_PATH, OUTPUT_FILE_PATH_DEFAULT); if (!outputFilePath.isEmpty()) { System.out.println("Raw data measurement: will output to result file: " + outputFilePath); try { outputStream = new PrintStream( new FileOutputStream(outputFilePath, true), true); } catch (FileNotFoundException e) { throw new RuntimeException("Failed to open raw data output file", e); } } else { System.out.println("Raw data measurement: will output to stdout."); outputStream = System.out; } noSummaryStats = Boolean.parseBoolean(props.getProperty(NO_SUMMARY_STATS, NO_SUMMARY_STATS_DEFAULT)); measurements = new LinkedList<>(); } @Override public synchronized void measure(int latency) { totalLatency += latency; windowTotalLatency += latency; windowOperations++; measurements.add(new RawDataPoint(latency)); } @Override public void exportMeasurements(MeasurementsExporter exporter) throws IOException { // Output raw data points first then print out a summary of percentiles to // stdout. outputStream.println(getName() + " latency raw data: op, timestamp(ms), latency(us)"); for (RawDataPoint point : measurements) { outputStream.println( String.format("%s,%d,%d", getName(), point.timeStamp(), point.value())); } if (outputStream != System.out) { outputStream.close(); } int totalOps = measurements.size(); exporter.write(getName(), "Total Operations", totalOps); if (totalOps > 0 && !noSummaryStats) { exporter.write(getName(), "Below is a summary of latency in microseconds:", -1); exporter.write(getName(), "Average", (double) totalLatency / (double) totalOps); Collections.sort(measurements, new RawDataPointComparator()); exporter.write(getName(), "Min", measurements.get(0).value()); exporter.write( getName(), "Max", measurements.get(totalOps - 1).value()); exporter.write( getName(), "p1", measurements.get((int) (totalOps * 0.01)).value()); exporter.write( getName(), "p5", measurements.get((int) (totalOps * 0.05)).value()); exporter.write( getName(), "p50", measurements.get((int) (totalOps * 0.5)).value()); exporter.write( getName(), "p90", measurements.get((int) (totalOps * 0.9)).value()); exporter.write( getName(), "p95", measurements.get((int) (totalOps * 0.95)).value()); exporter.write( getName(), "p99", measurements.get((int) (totalOps * 0.99)).value()); exporter.write(getName(), "p99.9", measurements.get((int) (totalOps * 0.999)).value()); exporter.write(getName(), "p99.99", measurements.get((int) (totalOps * 0.9999)).value()); } exportStatusCounts(exporter); } @Override public synchronized String getSummary() { if (windowOperations == 0) { return ""; } String toReturn = String.format("%s count: %d, average latency(us): %.2f", getName(), windowOperations, (double) windowTotalLatency / (double) windowOperations); windowTotalLatency = 0; windowOperations = 0; return toReturn; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import java.io.IOException; import java.text.DecimalFormat; import java.util.Properties; import java.util.Vector; class SeriesUnit { /** * @param time * @param average */ public SeriesUnit(long time, double average) { this.time = time; this.average = average; } protected final long time; protected final double average; } /** * A time series measurement of a metric, such as READ LATENCY. */ public class OneMeasurementTimeSeries extends OneMeasurement { /** * Granularity for time series; measurements will be averaged in chunks of this granularity. Units are milliseconds. */ public static final String GRANULARITY = "timeseries.granularity"; public static final String GRANULARITY_DEFAULT = "1000"; private final int granularity; private final Vector measurements; private long start = -1; private long currentunit = -1; private long count = 0; private long sum = 0; private long operations = 0; private long totallatency = 0; //keep a windowed version of these stats for printing status private int windowoperations = 0; private long windowtotallatency = 0; private int min = -1; private int max = -1; public OneMeasurementTimeSeries(String name, Properties props) { super(name); granularity = Integer.parseInt(props.getProperty(GRANULARITY, GRANULARITY_DEFAULT)); measurements = new Vector<>(); } private synchronized void checkEndOfUnit(boolean forceend) { long now = System.currentTimeMillis(); if (start < 0) { currentunit = 0; start = now; } long unit = ((now - start) / granularity) * granularity; if ((unit > currentunit) || (forceend)) { double avg = ((double) sum) / ((double) count); measurements.add(new SeriesUnit(currentunit, avg)); currentunit = unit; count = 0; sum = 0; } } @Override public void measure(int latency) { checkEndOfUnit(false); count++; sum += latency; totallatency += latency; operations++; windowoperations++; windowtotallatency += latency; if (latency > max) { max = latency; } if ((latency < min) || (min < 0)) { min = latency; } } @Override public void exportMeasurements(MeasurementsExporter exporter) throws IOException { checkEndOfUnit(true); exporter.write(getName(), "Operations", operations); exporter.write(getName(), "AverageLatency(us)", (((double) totallatency) / ((double) operations))); exporter.write(getName(), "MinLatency(us)", min); exporter.write(getName(), "MaxLatency(us)", max); // TODO: 95th and 99th percentile latency exportStatusCounts(exporter); for (SeriesUnit unit : measurements) { exporter.write(getName(), Long.toString(unit.time), unit.average); } } @Override public String getSummary() { if (windowoperations == 0) { return ""; } DecimalFormat d = new DecimalFormat("#.##"); double report = ((double) windowtotallatency) / ((double) windowoperations); windowtotallatency = 0; windowoperations = 0; return "[" + getName() + " AverageLatency(us)=" + d.format(report) + "]"; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/TwoInOneMeasurement.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; import java.io.IOException; /** * delegates to 2 measurement instances. */ public class TwoInOneMeasurement extends OneMeasurement { private final OneMeasurement thing1, thing2; public TwoInOneMeasurement(String name, OneMeasurement thing1, OneMeasurement thing2) { super(name); this.thing1 = thing1; this.thing2 = thing2; } /** * No need for synchronization, using CHM to deal with that. */ @Override public void reportStatus(final Status status) { thing1.reportStatus(status); } /** * It appears latency is reported in micros. * Using {@link org.HdrHistogram.Recorder} to support concurrent updates to histogram. */ @Override public void measure(int latencyInMicros) { thing1.measure(latencyInMicros); thing2.measure(latencyInMicros); } /** * This is called from a main thread, on orderly termination. */ @Override public void exportMeasurements(MeasurementsExporter exporter) throws IOException { thing1.exportMeasurements(exporter); thing2.exportMeasurements(exporter); } /** * This is called periodically from the StatusThread. There's a single StatusThread per Client process. * We optionally serialize the interval to log on this opportunity. * * @see com.yahoo.ycsb.measurements.OneMeasurement#getSummary() */ @Override public String getSummary() { return thing1.getSummary() + "\n" + thing2.getSummary(); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/exporter/JSONArrayMeasurementsExporter.java ================================================ /** * Copyright (c) 2015-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements.exporter; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.util.DefaultPrettyPrinter; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; /** * Export measurements into a machine readable JSON Array of measurement objects. */ public class JSONArrayMeasurementsExporter implements MeasurementsExporter { private final JsonFactory factory = new JsonFactory(); private JsonGenerator g; public JSONArrayMeasurementsExporter(OutputStream os) throws IOException { BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os)); g = factory.createJsonGenerator(bw); g.setPrettyPrinter(new DefaultPrettyPrinter()); g.writeStartArray(); } public void write(String metric, String measurement, int i) throws IOException { g.writeStartObject(); g.writeStringField("metric", metric); g.writeStringField("measurement", measurement); g.writeNumberField("value", i); g.writeEndObject(); } public void write(String metric, String measurement, long i) throws IOException { g.writeStartObject(); g.writeStringField("metric", metric); g.writeStringField("measurement", measurement); g.writeNumberField("value", i); g.writeEndObject(); } public void write(String metric, String measurement, double d) throws IOException { g.writeStartObject(); g.writeStringField("metric", metric); g.writeStringField("measurement", measurement); g.writeNumberField("value", d); g.writeEndObject(); } public void close() throws IOException { if (g != null) { g.writeEndArray(); g.close(); } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/exporter/JSONMeasurementsExporter.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements.exporter; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.util.DefaultPrettyPrinter; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; /** * Export measurements into a machine readable JSON file. */ public class JSONMeasurementsExporter implements MeasurementsExporter { private final JsonFactory factory = new JsonFactory(); private JsonGenerator g; public JSONMeasurementsExporter(OutputStream os) throws IOException { BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os)); g = factory.createJsonGenerator(bw); g.setPrettyPrinter(new DefaultPrettyPrinter()); } public void write(String metric, String measurement, int i) throws IOException { g.writeStartObject(); g.writeStringField("metric", metric); g.writeStringField("measurement", measurement); g.writeNumberField("value", i); g.writeEndObject(); } public void write(String metric, String measurement, long i) throws IOException { g.writeStartObject(); g.writeStringField("metric", metric); g.writeStringField("measurement", measurement); g.writeNumberField("value", i); g.writeEndObject(); } public void write(String metric, String measurement, double d) throws IOException { g.writeStartObject(); g.writeStringField("metric", metric); g.writeStringField("measurement", measurement); g.writeNumberField("value", d); g.writeEndObject(); } public void close() throws IOException { if (g != null) { g.close(); } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/exporter/MeasurementsExporter.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements.exporter; import java.io.Closeable; import java.io.IOException; /** * Used to export the collected measurements into a useful format, for example * human readable text or machine readable JSON. */ public interface MeasurementsExporter extends Closeable { /** * Write a measurement to the exported format. * * @param metric Metric name, for example "READ LATENCY". * @param measurement Measurement name, for example "Average latency". * @param i Measurement to write. * @throws IOException if writing failed */ void write(String metric, String measurement, int i) throws IOException; /** * Write a measurement to the exported format. * * @param metric Metric name, for example "READ LATENCY". * @param measurement Measurement name, for example "Average latency". * @param i Measurement to write. * @throws IOException if writing failed */ void write(String metric, String measurement, long i) throws IOException; /** * Write a measurement to the exported format. * * @param metric Metric name, for example "READ LATENCY". * @param measurement Measurement name, for example "Average latency". * @param d Measurement to write. * @throws IOException if writing failed */ void write(String metric, String measurement, double d) throws IOException; } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/exporter/TextMeasurementsExporter.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements.exporter; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; /** * Write human readable text. Tries to emulate the previous print report method. */ public class TextMeasurementsExporter implements MeasurementsExporter { private final BufferedWriter bw; public TextMeasurementsExporter(OutputStream os) { this.bw = new BufferedWriter(new OutputStreamWriter(os)); } public void write(String metric, String measurement, int i) throws IOException { bw.write("[" + metric + "], " + measurement + ", " + i); bw.newLine(); } public void write(String metric, String measurement, long i) throws IOException { bw.write("[" + metric + "], " + measurement + ", " + i); bw.newLine(); } public void write(String metric, String measurement, double d) throws IOException { bw.write("[" + metric + "], " + measurement + ", " + d); bw.newLine(); } public void close() throws IOException { this.bw.close(); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/exporter/package-info.java ================================================ /* * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB measurements.exporter package. */ package com.yahoo.ycsb.measurements.exporter; ================================================ FILE: core/src/main/java/com/yahoo/ycsb/measurements/package-info.java ================================================ /* * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB measurements package. */ package com.yahoo.ycsb.measurements; ================================================ FILE: core/src/main/java/com/yahoo/ycsb/package-info.java ================================================ /* * Copyright (c) 2015 - 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB core package. */ package com.yahoo.ycsb; ================================================ FILE: core/src/main/java/com/yahoo/ycsb/workloads/ConstantOccupancyWorkload.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.workloads; import com.yahoo.ycsb.Client; import com.yahoo.ycsb.WorkloadException; import com.yahoo.ycsb.generator.NumberGenerator; import java.util.Properties; /** * A disk-fragmenting workload. *

* Properties to control the client: *

*
    *
  • disksize: how many bytes of storage can the disk store? (default 100,000,000) *
  • occupancy: what fraction of the available storage should be used? (default 0.9) *
  • requestdistribution: what distribution should be used to select the records to operate on - uniform, * zipfian or latest (default: histogram) *
*

*

*

See also: * Russell Sears, Catharine van Ingen. * Fragmentation in Large Object * Repositories, * CIDR 2006. [Presentation] *

*/ public class ConstantOccupancyWorkload extends CoreWorkload { private long disksize; private long storageages; private double occupancy; private long objectCount; public static final String STORAGE_AGE_PROPERTY = "storageages"; public static final long STORAGE_AGE_PROPERTY_DEFAULT = 10; public static final String DISK_SIZE_PROPERTY = "disksize"; public static final long DISK_SIZE_PROPERTY_DEFAULT = 100 * 1000 * 1000; public static final String OCCUPANCY_PROPERTY = "occupancy"; public static final double OCCUPANCY_PROPERTY_DEFAULT = 0.9; @Override public void init(Properties p) throws WorkloadException { disksize = Long.parseLong(p.getProperty(DISK_SIZE_PROPERTY, String.valueOf(DISK_SIZE_PROPERTY_DEFAULT))); storageages = Long.parseLong(p.getProperty(STORAGE_AGE_PROPERTY, String.valueOf(STORAGE_AGE_PROPERTY_DEFAULT))); occupancy = Double.parseDouble(p.getProperty(OCCUPANCY_PROPERTY, String.valueOf(OCCUPANCY_PROPERTY_DEFAULT))); if (p.getProperty(Client.RECORD_COUNT_PROPERTY) != null || p.getProperty(Client.INSERT_COUNT_PROPERTY) != null || p.getProperty(Client.OPERATION_COUNT_PROPERTY) != null) { System.err.println("Warning: record, insert or operation count was set prior to initting " + "ConstantOccupancyWorkload. Overriding old values."); } NumberGenerator g = CoreWorkload.getFieldLengthGenerator(p); double fieldsize = g.mean(); int fieldcount = Integer.parseInt(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT)); objectCount = (long) (occupancy * (disksize / (fieldsize * fieldcount))); if (objectCount == 0) { throw new IllegalStateException("Object count was zero. Perhaps disksize is too low?"); } p.setProperty(Client.RECORD_COUNT_PROPERTY, String.valueOf(objectCount)); p.setProperty(Client.OPERATION_COUNT_PROPERTY, String.valueOf(storageages * objectCount)); p.setProperty(Client.INSERT_COUNT_PROPERTY, String.valueOf(objectCount)); super.init(p); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/workloads/CoreWorkload.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc., Copyright (c) 2016-2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.workloads; import com.yahoo.ycsb.*; import com.yahoo.ycsb.generator.*; import com.yahoo.ycsb.generator.UniformLongGenerator; import com.yahoo.ycsb.measurements.Measurements; import java.io.IOException; import java.util.*; /** * The core benchmark scenario. Represents a set of clients doing simple CRUD operations. The * relative proportion of different kinds of operations, and other properties of the workload, * are controlled by parameters specified at runtime. *

* Properties to control the client: *

    *
  • fieldcount: the number of fields in a record (default: 10) *
  • fieldlength: the size of each field (default: 100) *
  • readallfields: should reads read all fields (true) or just one (false) (default: true) *
  • writeallfields: should updates and read/modify/writes update all fields (true) or just * one (false) (default: false) *
  • readproportion: what proportion of operations should be reads (default: 0.95) *
  • updateproportion: what proportion of operations should be updates (default: 0.05) *
  • insertproportion: what proportion of operations should be inserts (default: 0) *
  • scanproportion: what proportion of operations should be scans (default: 0) *
  • readmodifywriteproportion: what proportion of operations should be read a record, * modify it, write it back (default: 0) *
  • requestdistribution: what distribution should be used to select the records to operate * on - uniform, zipfian, hotspot, sequential, exponential or latest (default: uniform) *
  • maxscanlength: for scans, what is the maximum number of records to scan (default: 1000) *
  • scanlengthdistribution: for scans, what distribution should be used to choose the * number of records to scan, for each scan, between 1 and maxscanlength (default: uniform) *
  • insertstart: for parallel loads and runs, defines the starting record for this * YCSB instance (default: 0) *
  • insertcount: for parallel loads and runs, defines the number of records for this * YCSB instance (default: recordcount) *
  • zeropadding: for generating a record sequence compatible with string sort order by * 0 padding the record number. Controls the number of 0s to use for padding. (default: 1) * For example for row 5, with zeropadding=1 you get 'user5' key and with zeropading=8 you get * 'user00000005' key. In order to see its impact, zeropadding needs to be bigger than number of * digits in the record number. *
  • insertorder: should records be inserted in order by key ("ordered"), or in hashed * order ("hashed") (default: hashed) *
*/ public class CoreWorkload extends Workload { /** * The name of the database table to run queries against. */ public static final String TABLENAME_PROPERTY = "table"; /** * The default name of the database table to run queries against. */ public static final String TABLENAME_PROPERTY_DEFAULT = "usertable"; protected String table; /** * The name of the property for the number of fields in a record. */ public static final String FIELD_COUNT_PROPERTY = "fieldcount"; /** * Default number of fields in a record. */ public static final String FIELD_COUNT_PROPERTY_DEFAULT = "10"; private List fieldnames; /** * The name of the property for the field length distribution. Options are "uniform", "zipfian" * (favouring short records), "constant", and "histogram". *

* If "uniform", "zipfian" or "constant", the maximum field length will be that specified by the * fieldlength property. If "histogram", then the histogram will be read from the filename * specified in the "fieldlengthhistogram" property. */ public static final String FIELD_LENGTH_DISTRIBUTION_PROPERTY = "fieldlengthdistribution"; /** * The default field length distribution. */ public static final String FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT = "constant"; /** * The name of the property for the length of a field in bytes. */ public static final String FIELD_LENGTH_PROPERTY = "fieldlength"; /** * The default maximum length of a field in bytes. */ public static final String FIELD_LENGTH_PROPERTY_DEFAULT = "100"; /** * The name of a property that specifies the filename containing the field length histogram (only * used if fieldlengthdistribution is "histogram"). */ public static final String FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY = "fieldlengthhistogram"; /** * The default filename containing a field length histogram. */ public static final String FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY_DEFAULT = "hist.txt"; /** * Generator object that produces field lengths. The value of this depends on the properties that * start with "FIELD_LENGTH_". */ protected NumberGenerator fieldlengthgenerator; /** * The name of the property for deciding whether to read one field (false) or all fields (true) of * a record. */ public static final String READ_ALL_FIELDS_PROPERTY = "readallfields"; /** * The default value for the readallfields property. */ public static final String READ_ALL_FIELDS_PROPERTY_DEFAULT = "true"; protected boolean readallfields; /** * The name of the property for deciding whether to write one field (false) or all fields (true) * of a record. */ public static final String WRITE_ALL_FIELDS_PROPERTY = "writeallfields"; /** * The default value for the writeallfields property. */ public static final String WRITE_ALL_FIELDS_PROPERTY_DEFAULT = "false"; protected boolean writeallfields; /** * The name of the property for deciding whether to check all returned * data against the formation template to ensure data integrity. */ public static final String DATA_INTEGRITY_PROPERTY = "dataintegrity"; /** * The default value for the dataintegrity property. */ public static final String DATA_INTEGRITY_PROPERTY_DEFAULT = "false"; /** * Set to true if want to check correctness of reads. Must also * be set to true during loading phase to function. */ private boolean dataintegrity; /** * The name of the property for the proportion of transactions that are reads. */ public static final String READ_PROPORTION_PROPERTY = "readproportion"; /** * The default proportion of transactions that are reads. */ public static final String READ_PROPORTION_PROPERTY_DEFAULT = "0.95"; /** * The name of the property for the proportion of transactions that are updates. */ public static final String UPDATE_PROPORTION_PROPERTY = "updateproportion"; /** * The default proportion of transactions that are updates. */ public static final String UPDATE_PROPORTION_PROPERTY_DEFAULT = "0.05"; /** * The name of the property for the proportion of transactions that are inserts. */ public static final String INSERT_PROPORTION_PROPERTY = "insertproportion"; /** * The default proportion of transactions that are inserts. */ public static final String INSERT_PROPORTION_PROPERTY_DEFAULT = "0.0"; /** * The name of the property for the proportion of transactions that are scans. */ public static final String SCAN_PROPORTION_PROPERTY = "scanproportion"; /** * The default proportion of transactions that are scans. */ public static final String SCAN_PROPORTION_PROPERTY_DEFAULT = "0.0"; /** * The name of the property for the proportion of transactions that are read-modify-write. */ public static final String READMODIFYWRITE_PROPORTION_PROPERTY = "readmodifywriteproportion"; /** * The default proportion of transactions that are scans. */ public static final String READMODIFYWRITE_PROPORTION_PROPERTY_DEFAULT = "0.0"; /** * The name of the property for the the distribution of requests across the keyspace. Options are * "uniform", "zipfian" and "latest" */ public static final String REQUEST_DISTRIBUTION_PROPERTY = "requestdistribution"; /** * The default distribution of requests across the keyspace. */ public static final String REQUEST_DISTRIBUTION_PROPERTY_DEFAULT = "uniform"; /** * The name of the property for adding zero padding to record numbers in order to match * string sort order. Controls the number of 0s to left pad with. */ public static final String ZERO_PADDING_PROPERTY = "zeropadding"; /** * The default zero padding value. Matches integer sort order */ public static final String ZERO_PADDING_PROPERTY_DEFAULT = "1"; /** * The name of the property for the max scan length (number of records). */ public static final String MAX_SCAN_LENGTH_PROPERTY = "maxscanlength"; /** * The default max scan length. */ public static final String MAX_SCAN_LENGTH_PROPERTY_DEFAULT = "1000"; /** * The name of the property for the scan length distribution. Options are "uniform" and "zipfian" * (favoring short scans) */ public static final String SCAN_LENGTH_DISTRIBUTION_PROPERTY = "scanlengthdistribution"; /** * The default max scan length. */ public static final String SCAN_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT = "uniform"; /** * The name of the property for the order to insert records. Options are "ordered" or "hashed" */ public static final String INSERT_ORDER_PROPERTY = "insertorder"; /** * Default insert order. */ public static final String INSERT_ORDER_PROPERTY_DEFAULT = "hashed"; /** * Percentage data items that constitute the hot set. */ public static final String HOTSPOT_DATA_FRACTION = "hotspotdatafraction"; /** * Default value of the size of the hot set. */ public static final String HOTSPOT_DATA_FRACTION_DEFAULT = "0.2"; /** * Percentage operations that access the hot set. */ public static final String HOTSPOT_OPN_FRACTION = "hotspotopnfraction"; /** * Default value of the percentage operations accessing the hot set. */ public static final String HOTSPOT_OPN_FRACTION_DEFAULT = "0.8"; /** * How many times to retry when insertion of a single item to a DB fails. */ public static final String INSERTION_RETRY_LIMIT = "core_workload_insertion_retry_limit"; public static final String INSERTION_RETRY_LIMIT_DEFAULT = "0"; /** * On average, how long to wait between the retries, in seconds. */ public static final String INSERTION_RETRY_INTERVAL = "core_workload_insertion_retry_interval"; public static final String INSERTION_RETRY_INTERVAL_DEFAULT = "3"; protected NumberGenerator keysequence; protected DiscreteGenerator operationchooser; protected NumberGenerator keychooser; protected NumberGenerator fieldchooser; protected AcknowledgedCounterGenerator transactioninsertkeysequence; protected NumberGenerator scanlength; protected boolean orderedinserts; protected long fieldcount; protected long recordcount; protected int zeropadding; protected int insertionRetryLimit; protected int insertionRetryInterval; private Measurements measurements = Measurements.getMeasurements(); protected static NumberGenerator getFieldLengthGenerator(Properties p) throws WorkloadException { NumberGenerator fieldlengthgenerator; String fieldlengthdistribution = p.getProperty( FIELD_LENGTH_DISTRIBUTION_PROPERTY, FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT); int fieldlength = Integer.parseInt(p.getProperty(FIELD_LENGTH_PROPERTY, FIELD_LENGTH_PROPERTY_DEFAULT)); String fieldlengthhistogram = p.getProperty( FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY, FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY_DEFAULT); if (fieldlengthdistribution.compareTo("constant") == 0) { fieldlengthgenerator = new ConstantIntegerGenerator(fieldlength); } else if (fieldlengthdistribution.compareTo("uniform") == 0) { fieldlengthgenerator = new UniformLongGenerator(1, fieldlength); } else if (fieldlengthdistribution.compareTo("zipfian") == 0) { fieldlengthgenerator = new ZipfianGenerator(1, fieldlength); } else if (fieldlengthdistribution.compareTo("histogram") == 0) { try { fieldlengthgenerator = new HistogramGenerator(fieldlengthhistogram); } catch (IOException e) { throw new WorkloadException( "Couldn't read field length histogram file: " + fieldlengthhistogram, e); } } else { throw new WorkloadException( "Unknown field length distribution \"" + fieldlengthdistribution + "\""); } return fieldlengthgenerator; } /** * Initialize the scenario. * Called once, in the main client thread, before any operations are started. */ @Override public void init(Properties p) throws WorkloadException { table = p.getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); fieldcount = Long.parseLong(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT)); fieldnames = new ArrayList<>(); for (int i = 0; i < fieldcount; i++) { fieldnames.add("field" + i); } fieldlengthgenerator = CoreWorkload.getFieldLengthGenerator(p); recordcount = Long.parseLong(p.getProperty(Client.RECORD_COUNT_PROPERTY, Client.DEFAULT_RECORD_COUNT)); if (recordcount == 0) { recordcount = Integer.MAX_VALUE; } String requestdistrib = p.getProperty(REQUEST_DISTRIBUTION_PROPERTY, REQUEST_DISTRIBUTION_PROPERTY_DEFAULT); int maxscanlength = Integer.parseInt(p.getProperty(MAX_SCAN_LENGTH_PROPERTY, MAX_SCAN_LENGTH_PROPERTY_DEFAULT)); String scanlengthdistrib = p.getProperty(SCAN_LENGTH_DISTRIBUTION_PROPERTY, SCAN_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT); long insertstart = Long.parseLong(p.getProperty(INSERT_START_PROPERTY, INSERT_START_PROPERTY_DEFAULT)); long insertcount= Integer.parseInt(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart))); // Confirm valid values for insertstart and insertcount in relation to recordcount if (recordcount < (insertstart + insertcount)) { System.err.println("Invalid combination of insertstart, insertcount and recordcount."); System.err.println("recordcount must be bigger than insertstart + insertcount."); System.exit(-1); } zeropadding = Integer.parseInt(p.getProperty(ZERO_PADDING_PROPERTY, ZERO_PADDING_PROPERTY_DEFAULT)); readallfields = Boolean.parseBoolean( p.getProperty(READ_ALL_FIELDS_PROPERTY, READ_ALL_FIELDS_PROPERTY_DEFAULT)); writeallfields = Boolean.parseBoolean( p.getProperty(WRITE_ALL_FIELDS_PROPERTY, WRITE_ALL_FIELDS_PROPERTY_DEFAULT)); dataintegrity = Boolean.parseBoolean( p.getProperty(DATA_INTEGRITY_PROPERTY, DATA_INTEGRITY_PROPERTY_DEFAULT)); // Confirm that fieldlengthgenerator returns a constant if data // integrity check requested. if (dataintegrity && !(p.getProperty( FIELD_LENGTH_DISTRIBUTION_PROPERTY, FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT)).equals("constant")) { System.err.println("Must have constant field size to check data integrity."); System.exit(-1); } if (p.getProperty(INSERT_ORDER_PROPERTY, INSERT_ORDER_PROPERTY_DEFAULT).compareTo("hashed") == 0) { orderedinserts = false; } else if (requestdistrib.compareTo("exponential") == 0) { double percentile = Double.parseDouble(p.getProperty( ExponentialGenerator.EXPONENTIAL_PERCENTILE_PROPERTY, ExponentialGenerator.EXPONENTIAL_PERCENTILE_DEFAULT)); double frac = Double.parseDouble(p.getProperty( ExponentialGenerator.EXPONENTIAL_FRAC_PROPERTY, ExponentialGenerator.EXPONENTIAL_FRAC_DEFAULT)); keychooser = new ExponentialGenerator(percentile, recordcount * frac); } else { orderedinserts = true; } keysequence = new CounterGenerator(insertstart); operationchooser = createOperationGenerator(p); transactioninsertkeysequence = new AcknowledgedCounterGenerator(recordcount); if (requestdistrib.compareTo("uniform") == 0) { keychooser = new UniformLongGenerator(insertstart, insertstart + insertcount - 1); } else if (requestdistrib.compareTo("sequential") == 0) { keychooser = new SequentialGenerator(insertstart, insertstart + insertcount - 1); } else if (requestdistrib.compareTo("zipfian") == 0) { // it does this by generating a random "next key" in part by taking the modulus over the // number of keys. // If the number of keys changes, this would shift the modulus, and we don't want that to // change which keys are popular so we'll actually construct the scrambled zipfian generator // with a keyspace that is larger than exists at the beginning of the test. that is, we'll predict // the number of inserts, and tell the scrambled zipfian generator the number of existing keys // plus the number of predicted keys as the total keyspace. then, if the generator picks a key // that hasn't been inserted yet, will just ignore it and pick another key. this way, the size of // the keyspace doesn't change from the perspective of the scrambled zipfian generator final double insertproportion = Double.parseDouble( p.getProperty(INSERT_PROPORTION_PROPERTY, INSERT_PROPORTION_PROPERTY_DEFAULT)); int opcount = Integer.parseInt(p.getProperty(Client.OPERATION_COUNT_PROPERTY)); int expectednewkeys = (int) ((opcount) * insertproportion * 2.0); // 2 is fudge factor keychooser = new ScrambledZipfianGenerator(insertstart, insertstart + insertcount + expectednewkeys); } else if (requestdistrib.compareTo("latest") == 0) { keychooser = new SkewedLatestGenerator(transactioninsertkeysequence); } else if (requestdistrib.equals("hotspot")) { double hotsetfraction = Double.parseDouble(p.getProperty(HOTSPOT_DATA_FRACTION, HOTSPOT_DATA_FRACTION_DEFAULT)); double hotopnfraction = Double.parseDouble(p.getProperty(HOTSPOT_OPN_FRACTION, HOTSPOT_OPN_FRACTION_DEFAULT)); keychooser = new HotspotIntegerGenerator(insertstart, insertstart + insertcount - 1, hotsetfraction, hotopnfraction); } else { throw new WorkloadException("Unknown request distribution \"" + requestdistrib + "\""); } fieldchooser = new UniformLongGenerator(0, fieldcount - 1); if (scanlengthdistrib.compareTo("uniform") == 0) { scanlength = new UniformLongGenerator(1, maxscanlength); } else if (scanlengthdistrib.compareTo("zipfian") == 0) { scanlength = new ZipfianGenerator(1, maxscanlength); } else { throw new WorkloadException( "Distribution \"" + scanlengthdistrib + "\" not allowed for scan length"); } insertionRetryLimit = Integer.parseInt(p.getProperty( INSERTION_RETRY_LIMIT, INSERTION_RETRY_LIMIT_DEFAULT)); insertionRetryInterval = Integer.parseInt(p.getProperty( INSERTION_RETRY_INTERVAL, INSERTION_RETRY_INTERVAL_DEFAULT)); } protected String buildKeyName(long keynum) { if (!orderedinserts) { keynum = Utils.hash(keynum); } String value = Long.toString(keynum); int fill = zeropadding - value.length(); String prekey = "user"; for (int i = 0; i < fill; i++) { prekey += '0'; } return prekey + value; } /** * Builds a value for a randomly chosen field. */ private HashMap buildSingleValue(String key) { HashMap value = new HashMap<>(); String fieldkey = fieldnames.get(fieldchooser.nextValue().intValue()); ByteIterator data; if (dataintegrity) { data = new StringByteIterator(buildDeterministicValue(key, fieldkey)); } else { // fill with random data data = new RandomByteIterator(fieldlengthgenerator.nextValue().longValue()); } value.put(fieldkey, data); return value; } /** * Builds values for all fields. */ private HashMap buildValues(String key) { HashMap values = new HashMap<>(); for (String fieldkey : fieldnames) { ByteIterator data; if (dataintegrity) { data = new StringByteIterator(buildDeterministicValue(key, fieldkey)); } else { // fill with random data data = new RandomByteIterator(fieldlengthgenerator.nextValue().longValue()); } values.put(fieldkey, data); } return values; } /** * Build a deterministic value given the key information. */ private String buildDeterministicValue(String key, String fieldkey) { int size = fieldlengthgenerator.nextValue().intValue(); StringBuilder sb = new StringBuilder(size); sb.append(key); sb.append(':'); sb.append(fieldkey); while (sb.length() < size) { sb.append(':'); sb.append(sb.toString().hashCode()); } sb.setLength(size); return sb.toString(); } /** * Do one insert operation. Because it will be called concurrently from multiple client threads, * this function must be thread safe. However, avoid synchronized, or the threads will block waiting * for each other, and it will be difficult to reach the target throughput. Ideally, this function would * have no side effects other than DB operations. */ @Override public boolean doInsert(DB db, Object threadstate) { int keynum = keysequence.nextValue().intValue(); String dbkey = buildKeyName(keynum); HashMap values = buildValues(dbkey); Status status; int numOfRetries = 0; do { status = db.insert(table, dbkey, values); if (null != status && status.isOk()) { break; } // Retry if configured. Without retrying, the load process will fail // even if one single insertion fails. User can optionally configure // an insertion retry limit (default is 0) to enable retry. if (++numOfRetries <= insertionRetryLimit) { System.err.println("Retrying insertion, retry count: " + numOfRetries); try { // Sleep for a random number between [0.8, 1.2)*insertionRetryInterval. int sleepTime = (int) (1000 * insertionRetryInterval * (0.8 + 0.4 * Math.random())); Thread.sleep(sleepTime); } catch (InterruptedException e) { break; } } else { System.err.println("Error inserting, not retrying any more. number of attempts: " + numOfRetries + "Insertion Retry Limit: " + insertionRetryLimit); break; } } while (true); return null != status && status.isOk(); } /** * Do one transaction operation. Because it will be called concurrently from multiple client * threads, this function must be thread safe. However, avoid synchronized, or the threads will block waiting * for each other, and it will be difficult to reach the target throughput. Ideally, this function would * have no side effects other than DB operations. */ @Override public boolean doTransaction(DB db, Object threadstate) { String operation = operationchooser.nextString(); if(operation == null) { return false; } switch (operation) { case "READ": doTransactionRead(db); break; case "UPDATE": doTransactionUpdate(db); break; case "INSERT": doTransactionInsert(db); break; case "SCAN": doTransactionScan(db); break; default: doTransactionReadModifyWrite(db); } return true; } /** * Results are reported in the first three buckets of the histogram under * the label "VERIFY". * Bucket 0 means the expected data was returned. * Bucket 1 means incorrect data was returned. * Bucket 2 means null data was returned when some data was expected. */ protected void verifyRow(String key, HashMap cells) { Status verifyStatus = Status.OK; long startTime = System.nanoTime(); if (!cells.isEmpty()) { for (Map.Entry entry : cells.entrySet()) { if (!entry.getValue().toString().equals(buildDeterministicValue(key, entry.getKey()))) { verifyStatus = Status.UNEXPECTED_STATE; break; } } } else { // This assumes that null data is never valid verifyStatus = Status.ERROR; } long endTime = System.nanoTime(); measurements.measure("VERIFY", (int) (endTime - startTime) / 1000); measurements.reportStatus("VERIFY", verifyStatus); } long nextKeynum() { long keynum; if (keychooser instanceof ExponentialGenerator) { do { keynum = transactioninsertkeysequence.lastValue() - keychooser.nextValue().intValue(); } while (keynum < 0); } else { do { keynum = keychooser.nextValue().intValue(); } while (keynum > transactioninsertkeysequence.lastValue()); } return keynum; } public void doTransactionRead(DB db) { // choose a random key long keynum = nextKeynum(); String keyname = buildKeyName(keynum); HashSet fields = null; if (!readallfields) { // read a random field String fieldname = fieldnames.get(fieldchooser.nextValue().intValue()); fields = new HashSet(); fields.add(fieldname); } else if (dataintegrity) { // pass the full field list if dataintegrity is on for verification fields = new HashSet(fieldnames); } HashMap cells = new HashMap(); db.read(table, keyname, fields, cells); if (dataintegrity) { verifyRow(keyname, cells); } } public void doTransactionReadModifyWrite(DB db) { // choose a random key long keynum = nextKeynum(); String keyname = buildKeyName(keynum); HashSet fields = null; if (!readallfields) { // read a random field String fieldname = fieldnames.get(fieldchooser.nextValue().intValue()); fields = new HashSet(); fields.add(fieldname); } HashMap values; if (writeallfields) { // new data for all the fields values = buildValues(keyname); } else { // update a random field values = buildSingleValue(keyname); } // do the transaction HashMap cells = new HashMap(); long ist = measurements.getIntendedtartTimeNs(); long st = System.nanoTime(); db.read(table, keyname, fields, cells); db.update(table, keyname, values); long en = System.nanoTime(); if (dataintegrity) { verifyRow(keyname, cells); } measurements.measure("READ-MODIFY-WRITE", (int) ((en - st) / 1000)); measurements.measureIntended("READ-MODIFY-WRITE", (int) ((en - ist) / 1000)); } public void doTransactionScan(DB db) { // choose a random key long keynum = nextKeynum(); String startkeyname = buildKeyName(keynum); // choose a random scan length int len = scanlength.nextValue().intValue(); HashSet fields = null; if (!readallfields) { // read a random field String fieldname = fieldnames.get(fieldchooser.nextValue().intValue()); fields = new HashSet(); fields.add(fieldname); } db.scan(table, startkeyname, len, fields, new Vector>()); } public void doTransactionUpdate(DB db) { // choose a random key long keynum = nextKeynum(); String keyname = buildKeyName(keynum); HashMap values; if (writeallfields) { // new data for all the fields values = buildValues(keyname); } else { // update a random field values = buildSingleValue(keyname); } db.update(table, keyname, values); } public void doTransactionInsert(DB db) { // choose the next key long keynum = transactioninsertkeysequence.nextValue(); try { String dbkey = buildKeyName(keynum); HashMap values = buildValues(dbkey); db.insert(table, dbkey, values); } finally { transactioninsertkeysequence.acknowledge(keynum); } } /** * Creates a weighted discrete values with database operations for a workload to perform. * Weights/proportions are read from the properties list and defaults are used * when values are not configured. * Current operations are "READ", "UPDATE", "INSERT", "SCAN" and "READMODIFYWRITE". * * @param p The properties list to pull weights from. * @return A generator that can be used to determine the next operation to perform. * @throws IllegalArgumentException if the properties object was null. */ protected static DiscreteGenerator createOperationGenerator(final Properties p) { if (p == null) { throw new IllegalArgumentException("Properties object cannot be null"); } final double readproportion = Double.parseDouble( p.getProperty(READ_PROPORTION_PROPERTY, READ_PROPORTION_PROPERTY_DEFAULT)); final double updateproportion = Double.parseDouble( p.getProperty(UPDATE_PROPORTION_PROPERTY, UPDATE_PROPORTION_PROPERTY_DEFAULT)); final double insertproportion = Double.parseDouble( p.getProperty(INSERT_PROPORTION_PROPERTY, INSERT_PROPORTION_PROPERTY_DEFAULT)); final double scanproportion = Double.parseDouble( p.getProperty(SCAN_PROPORTION_PROPERTY, SCAN_PROPORTION_PROPERTY_DEFAULT)); final double readmodifywriteproportion = Double.parseDouble(p.getProperty( READMODIFYWRITE_PROPORTION_PROPERTY, READMODIFYWRITE_PROPORTION_PROPERTY_DEFAULT)); final DiscreteGenerator operationchooser = new DiscreteGenerator(); if (readproportion > 0) { operationchooser.addValue(readproportion, "READ"); } if (updateproportion > 0) { operationchooser.addValue(updateproportion, "UPDATE"); } if (insertproportion > 0) { operationchooser.addValue(insertproportion, "INSERT"); } if (scanproportion > 0) { operationchooser.addValue(scanproportion, "SCAN"); } if (readmodifywriteproportion > 0) { operationchooser.addValue(readmodifywriteproportion, "READMODIFYWRITE"); } return operationchooser; } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/workloads/RestWorkload.java ================================================ /** * Copyright (c) 2016-2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.workloads; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.RandomByteIterator; import com.yahoo.ycsb.WorkloadException; import com.yahoo.ycsb.generator.*; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Properties; import com.yahoo.ycsb.generator.UniformLongGenerator; /** * Typical RESTFul services benchmarking scenario. Represents a set of client * calling REST operations like HTTP DELETE, GET, POST, PUT on a web service. * This scenario is completely different from CoreWorkload which is mainly * designed for databases benchmarking. However due to some reusable * functionality this class extends {@link CoreWorkload} and overrides necessary * methods like init, doTransaction etc. */ public class RestWorkload extends CoreWorkload { /** * The name of the property for the proportion of transactions that are * delete. */ public static final String DELETE_PROPORTION_PROPERTY = "deleteproportion"; /** * The default proportion of transactions that are delete. */ public static final String DELETE_PROPORTION_PROPERTY_DEFAULT = "0.00"; /** * The name of the property for the file that holds the field length size for insert operations. */ public static final String FIELD_LENGTH_DISTRIBUTION_FILE_PROPERTY = "fieldlengthdistfile"; /** * The default file name that holds the field length size for insert operations. */ public static final String FIELD_LENGTH_DISTRIBUTION_FILE_PROPERTY_DEFAULT = "fieldLengthDistFile.txt"; /** * In web services even though the CRUD operations follow the same request * distribution, they have different traces and distribution parameter * values. Hence configuring the parameters of these operations separately * makes the benchmark more flexible and capable of generating better * realistic workloads. */ // Read related properties. private static final String READ_TRACE_FILE = "url.trace.read"; private static final String READ_TRACE_FILE_DEFAULT = "readtrace.txt"; private static final String READ_ZIPFIAN_CONSTANT = "readzipfconstant"; private static final String READ_ZIPFIAN_CONSTANT_DEAFULT = "0.99"; private static final String READ_RECORD_COUNT_PROPERTY = "readrecordcount"; // Insert related properties. private static final String INSERT_TRACE_FILE = "url.trace.insert"; private static final String INSERT_TRACE_FILE_DEFAULT = "inserttrace.txt"; private static final String INSERT_ZIPFIAN_CONSTANT = "insertzipfconstant"; private static final String INSERT_ZIPFIAN_CONSTANT_DEAFULT = "0.99"; private static final String INSERT_SIZE_ZIPFIAN_CONSTANT = "insertsizezipfconstant"; private static final String INSERT_SIZE_ZIPFIAN_CONSTANT_DEAFULT = "0.99"; private static final String INSERT_RECORD_COUNT_PROPERTY = "insertrecordcount"; // Delete related properties. private static final String DELETE_TRACE_FILE = "url.trace.delete"; private static final String DELETE_TRACE_FILE_DEFAULT = "deletetrace.txt"; private static final String DELETE_ZIPFIAN_CONSTANT = "deletezipfconstant"; private static final String DELETE_ZIPFIAN_CONSTANT_DEAFULT = "0.99"; private static final String DELETE_RECORD_COUNT_PROPERTY = "deleterecordcount"; // Delete related properties. private static final String UPDATE_TRACE_FILE = "url.trace.update"; private static final String UPDATE_TRACE_FILE_DEFAULT = "updatetrace.txt"; private static final String UPDATE_ZIPFIAN_CONSTANT = "updatezipfconstant"; private static final String UPDATE_ZIPFIAN_CONSTANT_DEAFULT = "0.99"; private static final String UPDATE_RECORD_COUNT_PROPERTY = "updaterecordcount"; private Map readUrlMap; private Map insertUrlMap; private Map deleteUrlMap; private Map updateUrlMap; private int readRecordCount; private int insertRecordCount; private int deleteRecordCount; private int updateRecordCount; private NumberGenerator readKeyChooser; private NumberGenerator insertKeyChooser; private NumberGenerator deleteKeyChooser; private NumberGenerator updateKeyChooser; private NumberGenerator fieldlengthgenerator; private DiscreteGenerator operationchooser; @Override public void init(Properties p) throws WorkloadException { readRecordCount = Integer.parseInt(p.getProperty(READ_RECORD_COUNT_PROPERTY, String.valueOf(Integer.MAX_VALUE))); insertRecordCount = Integer .parseInt(p.getProperty(INSERT_RECORD_COUNT_PROPERTY, String.valueOf(Integer.MAX_VALUE))); deleteRecordCount = Integer .parseInt(p.getProperty(DELETE_RECORD_COUNT_PROPERTY, String.valueOf(Integer.MAX_VALUE))); updateRecordCount = Integer .parseInt(p.getProperty(UPDATE_RECORD_COUNT_PROPERTY, String.valueOf(Integer.MAX_VALUE))); readUrlMap = getTrace(p.getProperty(READ_TRACE_FILE, READ_TRACE_FILE_DEFAULT), readRecordCount); insertUrlMap = getTrace(p.getProperty(INSERT_TRACE_FILE, INSERT_TRACE_FILE_DEFAULT), insertRecordCount); deleteUrlMap = getTrace(p.getProperty(DELETE_TRACE_FILE, DELETE_TRACE_FILE_DEFAULT), deleteRecordCount); updateUrlMap = getTrace(p.getProperty(UPDATE_TRACE_FILE, UPDATE_TRACE_FILE_DEFAULT), updateRecordCount); operationchooser = createOperationGenerator(p); // Common distribution for all operations. String requestDistrib = p.getProperty(REQUEST_DISTRIBUTION_PROPERTY, REQUEST_DISTRIBUTION_PROPERTY_DEFAULT); double readZipfconstant = Double.parseDouble(p.getProperty(READ_ZIPFIAN_CONSTANT, READ_ZIPFIAN_CONSTANT_DEAFULT)); readKeyChooser = getKeyChooser(requestDistrib, readUrlMap.size(), readZipfconstant, p); double updateZipfconstant = Double .parseDouble(p.getProperty(UPDATE_ZIPFIAN_CONSTANT, UPDATE_ZIPFIAN_CONSTANT_DEAFULT)); updateKeyChooser = getKeyChooser(requestDistrib, updateUrlMap.size(), updateZipfconstant, p); double insertZipfconstant = Double .parseDouble(p.getProperty(INSERT_ZIPFIAN_CONSTANT, INSERT_ZIPFIAN_CONSTANT_DEAFULT)); insertKeyChooser = getKeyChooser(requestDistrib, insertUrlMap.size(), insertZipfconstant, p); double deleteZipfconstant = Double .parseDouble(p.getProperty(DELETE_ZIPFIAN_CONSTANT, DELETE_ZIPFIAN_CONSTANT_DEAFULT)); deleteKeyChooser = getKeyChooser(requestDistrib, deleteUrlMap.size(), deleteZipfconstant, p); fieldlengthgenerator = getFieldLengthGenerator(p); } public static DiscreteGenerator createOperationGenerator(final Properties p) { // Re-using CoreWorkload method. final DiscreteGenerator operationChooser = CoreWorkload.createOperationGenerator(p); // Needs special handling for delete operations not supported in CoreWorkload. double deleteproportion = Double .parseDouble(p.getProperty(DELETE_PROPORTION_PROPERTY, DELETE_PROPORTION_PROPERTY_DEFAULT)); if (deleteproportion > 0) { operationChooser.addValue(deleteproportion, "DELETE"); } return operationChooser; } private static NumberGenerator getKeyChooser(String requestDistrib, int recordCount, double zipfContant, Properties p) throws WorkloadException { NumberGenerator keychooser; switch (requestDistrib) { case "exponential": double percentile = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_PERCENTILE_PROPERTY, ExponentialGenerator.EXPONENTIAL_PERCENTILE_DEFAULT)); double frac = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_FRAC_PROPERTY, ExponentialGenerator.EXPONENTIAL_FRAC_DEFAULT)); keychooser = new ExponentialGenerator(percentile, recordCount * frac); break; case "uniform": keychooser = new UniformLongGenerator(0, recordCount - 1); break; case "zipfian": keychooser = new ZipfianGenerator(recordCount, zipfContant); break; case "latest": throw new WorkloadException("Latest request distribution is not supported for RestWorkload."); case "hotspot": double hotsetfraction = Double.parseDouble(p.getProperty(HOTSPOT_DATA_FRACTION, HOTSPOT_DATA_FRACTION_DEFAULT)); double hotopnfraction = Double.parseDouble(p.getProperty(HOTSPOT_OPN_FRACTION, HOTSPOT_OPN_FRACTION_DEFAULT)); keychooser = new HotspotIntegerGenerator(0, recordCount - 1, hotsetfraction, hotopnfraction); break; default: throw new WorkloadException("Unknown request distribution \"" + requestDistrib + "\""); } return keychooser; } protected static NumberGenerator getFieldLengthGenerator(Properties p) throws WorkloadException { // Re-using CoreWorkload method. NumberGenerator fieldLengthGenerator = CoreWorkload.getFieldLengthGenerator(p); String fieldlengthdistribution = p.getProperty(FIELD_LENGTH_DISTRIBUTION_PROPERTY, FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT); // Needs special handling for Zipfian distribution for variable Zipf Constant. if (fieldlengthdistribution.compareTo("zipfian") == 0) { int fieldlength = Integer.parseInt(p.getProperty(FIELD_LENGTH_PROPERTY, FIELD_LENGTH_PROPERTY_DEFAULT)); double insertsizezipfconstant = Double .parseDouble(p.getProperty(INSERT_SIZE_ZIPFIAN_CONSTANT, INSERT_SIZE_ZIPFIAN_CONSTANT_DEAFULT)); fieldLengthGenerator = new ZipfianGenerator(1, fieldlength, insertsizezipfconstant); } return fieldLengthGenerator; } /** * Reads the trace file and returns a URL map. */ private static Map getTrace(String filePath, int recordCount) throws WorkloadException { Map urlMap = new HashMap(); int count = 0; String line; try { FileReader inputFile = new FileReader(filePath); BufferedReader bufferReader = new BufferedReader(inputFile); while ((line = bufferReader.readLine()) != null) { urlMap.put(count++, line.trim()); if (count >= recordCount) { break; } } bufferReader.close(); } catch (IOException e) { throw new WorkloadException( "Error while reading the trace. Please make sure the trace file path is correct. " + e.getLocalizedMessage()); } return urlMap; } /** * Not required for Rest Clients as data population is service specific. */ @Override public boolean doInsert(DB db, Object threadstate) { return false; } @Override public boolean doTransaction(DB db, Object threadstate) { String operation = operationchooser.nextString(); if (operation == null) { return false; } switch (operation) { case "UPDATE": doTransactionUpdate(db); break; case "INSERT": doTransactionInsert(db); break; case "DELETE": doTransactionDelete(db); break; default: doTransactionRead(db); } return true; } /** * Returns next URL to be called. */ private String getNextURL(int opType) { if (opType == 1) { return readUrlMap.get(readKeyChooser.nextValue().intValue()); } else if (opType == 2) { return insertUrlMap.get(insertKeyChooser.nextValue().intValue()); } else if (opType == 3) { return deleteUrlMap.get(deleteKeyChooser.nextValue().intValue()); } else { return updateUrlMap.get(updateKeyChooser.nextValue().intValue()); } } @Override public void doTransactionRead(DB db) { HashMap result = new HashMap(); db.read(null, getNextURL(1), null, result); } @Override public void doTransactionInsert(DB db) { HashMap value = new HashMap(); // Create random bytes of insert data with a specific size. value.put("data", new RandomByteIterator(fieldlengthgenerator.nextValue().longValue())); db.insert(null, getNextURL(2), value); } public void doTransactionDelete(DB db) { db.delete(null, getNextURL(3)); } @Override public void doTransactionUpdate(DB db) { HashMap value = new HashMap(); // Create random bytes of update data with a specific size. value.put("data", new RandomByteIterator(fieldlengthgenerator.nextValue().longValue())); db.update(null, getNextURL(4), value); } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/workloads/TimeSeriesWorkload.java ================================================ /** * Copyright (c) 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.workloads; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.Vector; import java.util.concurrent.TimeUnit; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Client; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.NumericByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.Utils; import com.yahoo.ycsb.Workload; import com.yahoo.ycsb.WorkloadException; import com.yahoo.ycsb.generator.DiscreteGenerator; import com.yahoo.ycsb.generator.Generator; import com.yahoo.ycsb.generator.HotspotIntegerGenerator; import com.yahoo.ycsb.generator.IncrementingPrintableStringGenerator; import com.yahoo.ycsb.generator.NumberGenerator; import com.yahoo.ycsb.generator.RandomDiscreteTimestampGenerator; import com.yahoo.ycsb.generator.ScrambledZipfianGenerator; import com.yahoo.ycsb.generator.SequentialGenerator; import com.yahoo.ycsb.generator.UniformLongGenerator; import com.yahoo.ycsb.generator.UnixEpochTimestampGenerator; import com.yahoo.ycsb.generator.ZipfianGenerator; import com.yahoo.ycsb.measurements.Measurements; /** * A specialized workload dealing with time series data, i.e. series of discreet * events associated with timestamps and identifiers. For this workload, identities * consist of a {@link String} key and a set of {@link String} tag key/value * pairs. *

* For example: * * * * * * *
Time Series KeyTag Keys/Values148322880014832288601483228920
AAAA=AA, AB=AA42.51.085.9
AAAA=AA, AB=AB-9.476.90.18
ABAA=AA, AB=AA-93.057.1-63.8
ABAA=AA, AB=AB7.656.1-0.3
*

* This table shows four time series with 3 measurements at three different timestamps. * Keys, tags, timestamps and values (numeric only at this time) are generated by * this workload. For details on properties and behavior, see the * {@code workloads/tsworkload_template} file. The Javadocs will focus on implementation * and how {@link DB} clients can parse the workload. *

* In order to avoid having existing DB implementations implement a brand new interface * this workload uses the existing APIs to encode a few special values that can be parsed * by the client. The special values include the timestamp, numeric value and some * query (read or scan) parameters. As an example on how to parse the fields, see * {@link BasicTSDB}. *

* Timestamps *

* Timestamps are presented as Unix Epoch values in units of {@link TimeUnit#SECONDS}, * {@link TimeUnit#MILLISECONDS} or {@link TimeUnit#NANOSECONDS} based on the * {@code timestampunits} property. For calls to {@link DB#insert(String, String, java.util.Map)} * and {@link DB#update(String, String, java.util.Map)}, the timestamp is added to the * {@code values} map encoded in a {@link NumericByteIterator} with the key defined * in the {@code timestampkey} property (defaulting to "YCSBTS"). To pull out the timestamp * when iterating over the values map, cast the {@link ByteIterator} to a * {@link NumericByteIterator} and call {@link NumericByteIterator#getLong()}. *

* Note that for calls to {@link DB#update(String, String, java.util.Map)}, timestamps * earlier than the timestamp generator's timestamp will be choosen at random to * mimic a lambda architecture or old job re-reporting some data. *

* For calls to {@link DB#read(String, String, java.util.Set, java.util.Map)} and * {@link DB#scan(String, String, int, java.util.Set, Vector)}, timestamps * are encoded in a {@link StringByteIterator} in a key/value format with the * {@code tagpairdelimiter} separator. E.g {@code YCSBTS=1483228800}. If {@code querytimespan} * has been set to a positive value then the value will include a range with the * starting (oldest) timestamp followed by the {@code querytimespandelimiter} separator * and the ending (most recent) timestamp. E.g. {@code YCSBTS=1483228800-1483228920}. *

* For calls to {@link DB#delete(String, String)}, encoding is the same as reads and * scans but key/value pairs are separated by the {@code deletedelimiter} property value. *

* By default, the starting timestamp is the current system time without any rounding. * All timestamps are then offsets from that starting value. *

* Values *

* Similar to timestamps, values are encoded in {@link NumericByteIterator}s and stored * in the values map with the key defined in {@code valuekey} (defaulting to "YCSBV"). * Values can either be 64 bit signed {@link long}s or double precision {@link double}s * depending on the {@code valuetype} or {@code dataintegrity} properties. When parsing * out the value, always call {@link NumericByteIterator#isFloatingPoint()} to determine * whether or not to call {@link NumericByteIterator#getDouble()} (true) or * {@link NumericByteIterator#getLong()} (false). *

* When {@code dataintegrity} is set to true, then the value is always set to a * 64 bit signed integer which is the Java hash code of the concatenation of the * key and map of values (sorted on the map keys and skipping the timestamp and value * entries) OR'd with the timestamp of the data point. See * {@link #validationFunction(String, long, TreeMap)} for the implementation. *

* Keys and Tags *

* As mentioned, the workload generates strings for the keys and tags. On initialization * three string generators are created using the {@link IncrementingPrintableStringGenerator} * implementation. Then the generators fill three arrays with values based on the * number of keys, the number of tags and the cardinality of each tag key/value pair. * This implementation gives us time series like the example table where every string * starts at something like "AA" (depending on the length of keys, tag keys and tag values) * and continuing to "ZZ" wherein they rollover back to "AA". *

* Each time series must have a unique set of tag keys, i.e. the key "AA" cannot appear * more than once per time series. If the workload is configured for four tags with a * tag key length of 2, the keys would be "AA", "AB", "AC" and "AD". *

* Each tag key is then associated with a tag value. Tag values may appear more than once * in each time series. E.g. time series will usually start with the tags "AA=AA", * "AB=AA", "AC=AA" and "AD=AA". The {@code tagcardinality} property determines how many * unique values will be generated per tag key. In the example table above, the * {@code tagcardinality} property would have been set to {@code 1,2} meaning tag * key "AA" would always have the tag value "AA" given a cardinality of 1. However * tag key "AB" would have values "AA" and "AB" due to a cardinality of 2. This * cardinality map, along with the number of unique time series keys determines how * many unique time series are generated for the workload. Tag values share a common * array of generated strings to save on memory. *

* Operation Order *

* The default behavior of the workload (for inserts and updates) is to generate a * value for each time series for a given timestamp before incrementing to the next * timestamp and writing values. This is an ideal workload and some time series * databases are designed for this behavior. However in the real-world events will * arrive grouped close to the current system time with a number of events being * delayed, hence their timestamps are further in the past. The {@code delayedseries} * property determines the percentage of time series that are delayed by up to * {@code delayedintervals} intervals. E.g. setting this value to 0.05 means that * 5% of the time series will be written with timestamps earlier than the timestamp * generator's current time. *

* Reads and Scans *

* For benchmarking queries, some common tasks implemented by almost every time series * data base are available and are passed in the fields {@link Set}: *

* GroupBy - A common operation is to aggregate multiple time series into a * single time series via common parameters. For example, a user may want to see the * total network traffic in a data center so they'll issue a SQL query like: * SELECT value FROM timeseriesdb GROUP BY datacenter ORDER BY SUM(value); * If the {@code groupbyfunction} has been set to a group by function, then the fields * will contain a key/value pair with the key set in {@code groupbykey}. E.g. * {@code YCSBGB=SUM}. *

* Additionally with grouping enabled, fields on tag keys where group bys should * occur will only have the key defined and will not have a value or delimiter. E.g. * if grouping on tag key "AA", the field will contain {@code AA} instead of {@code AA=AB}. *

* Downsampling - Another common operation is to reduce the resolution of the * queried time series when fetching a wide time range of data so fewer data points * are returned. For example, a user may fetch a week of data but if the data is * recorded on a 1 second interval, that would be over 600k data points so they * may ask for a 1 hour downsampling (also called bucketing) wherein every hour, all * of the data points for a "bucket" are aggregated into a single value. *

* To enable downsampling, the {@code downsamplingfunction} property must be set to * a supported function such as "SUM" and the {@code downsamplinginterval} must be * set to a valid time interval with the same units as {@code timestampunits}, e.g. * "3600" which would create 1 hour buckets if the time units were set to seconds. * With downsampling, query fields will include a key/value pair with * {@code downsamplingkey} as the key (defaulting to "YCSBDS") and the value being * a concatenation of {@code downsamplingfunction} and {@code downsamplinginterval}, * for example {@code YCSBDS=SUM60}. *

* Timestamps - For every read, a random timestamp is selected from the interval * set. If {@code querytimespan} has been set to a positive value, then the configured * query time interval is added to the selected timestamp so the read passes the DB * a range of times. Note that during the run phase, if no data was previously loaded, * or if there are more {@code recordcount}s set for the run phase, reads may be sent * to the DB with timestamps that are beyond the written data time range (or even the * system clock of the DB). *

* Deletes *

* Because the delete API only accepts a single key, a full key and tag key/value * pair map is flattened into a single string for parsing by the database. Common * workloads include deleting a single time series (wherein all tag key and values are * defined), deleting all series containing a tag key and value or deleting all of the * time series sharing a common time series key. *

* Right now the workload supports deletes with a key and for time series tag key/value * pairs or a key with tags and a group by on one or more tags (meaning, delete all of * the series with any value for the given tag key). The parameters are collapsed into * a single string delimited with the character in the {@code deletedelimiter} property. * For example, a delete request may look like: {@code AA:AA=AA:AA=AB} to delete the * first time series in the table above. *

* Threads *

* For a multi-threaded execution, the number of time series keys set via the * {@code fieldcount} property, must be greater than or equal to the number of * threads set via {@code threads}. This is due to each thread choosing a subset * of the total number of time series keys and being responsible for writing values * for each time series containing those keys at each timestamp. Thus each thread * will have it's own timestamp generator, incrementing each time every time series * it is responsible for has had a value written. *

* Each thread may, however, issue reads and scans for any time series in the * complete set. *

* Sparsity *

* By default, during loads, every time series will have a data point written at every * time stamp in the interval set. This is common in workloads where a sensor writes * a value at regular intervals. However some time series are only reported under * certain conditions. *

* For example, a counter may track the number of errors over a * time period for a web service and only report when the value is greater than 1. * Or a time series may include tags such as a user ID and IP address when a request * arrives at the web service and only report values when that combination is seen. * This means the timeseries will not have a value at every timestamp and in * some cases there may be only a single value! *

* This workload has a {@code sparsity} parameter that can choose how often a * time series should record a value. The default value of 0.0 means every series * will get a value at every timestamp. A value of 0.95 will mean that for each * series, only 5% of the timestamps in the interval will have a value. The distribution * of values is random. *

* Notes/Warnings *

*

    *
  • Because time series keys and tag key/values are generated and stored in memory, * be careful of setting the cardinality too high for the JVM's heap.
    • *
  • When running for data integrity, a number of settings are incompatible and will * throw errors. Check the error messages for details.
    • *
  • Databases that support keys only and can't store tags should order and then * collapse the tag values using a delimiter. For example the series in the example * table at the top could be written as: *
      *
    • {@code AA.AA.AA}
      • *
    • {@code AA.AA.AB}
      • *
    • {@code AB.AA.AA}
      • *
    • {@code AB.AA.AB}
      • *
    • *
*

* TODOs *

*

    *
  • Support random time intervals. E.g. some series write every second, others every * 60 seconds.
    • *
  • Support random time series cardinality. Right now every series has the same * cardinality.
    • *
  • Truly random timetamps per time series. We could use bitmaps to determine if * a series has had a value written for a given timestamp. Right now all of the series * are in sync time-wise.
    • *
  • Possibly a real-time load where values are written with the current system time. * It's more of a bulk-loading operation now.
    • *
*/ public class TimeSeriesWorkload extends Workload { /** * The types of values written to the timeseries store. */ public enum ValueType { INTEGERS("integers"), FLOATS("floats"), MIXED("mixednumbers"); protected final String name; ValueType(final String name) { this.name = name; } public static ValueType fromString(final String name) { for (final ValueType type : ValueType.values()) { if (type.name.equalsIgnoreCase(name)) { return type; } } throw new IllegalArgumentException("Unrecognized type: " + name); } } /** Name and default value for the timestamp key property. */ public static final String TIMESTAMP_KEY_PROPERTY = "timestampkey"; public static final String TIMESTAMP_KEY_PROPERTY_DEFAULT = "YCSBTS"; /** Name and default value for the value key property. */ public static final String VALUE_KEY_PROPERTY = "valuekey"; public static final String VALUE_KEY_PROPERTY_DEFAULT = "YCSBV"; /** Name and default value for the timestamp interval property. */ public static final String TIMESTAMP_INTERVAL_PROPERTY = "timestampinterval"; public static final String TIMESTAMP_INTERVAL_PROPERTY_DEFAULT = "60"; /** Name and default value for the timestamp units property. */ public static final String TIMESTAMP_UNITS_PROPERTY = "timestampunits"; public static final String TIMESTAMP_UNITS_PROPERTY_DEFAULT = "SECONDS"; /** Name and default value for the number of tags property. */ public static final String TAG_COUNT_PROPERTY = "tagcount"; public static final String TAG_COUNT_PROPERTY_DEFAULT = "4"; /** Name and default value for the tag value cardinality map property. */ public static final String TAG_CARDINALITY_PROPERTY = "tagcardinality"; public static final String TAG_CARDINALITY_PROPERTY_DEFAULT = "1, 2, 4, 8"; /** Name and default value for the tag key length property. */ public static final String TAG_KEY_LENGTH_PROPERTY = "tagkeylength"; public static final String TAG_KEY_LENGTH_PROPERTY_DEFAULT = "8"; /** Name and default value for the tag value length property. */ public static final String TAG_VALUE_LENGTH_PROPERTY = "tagvaluelength"; public static final String TAG_VALUE_LENGTH_PROPERTY_DEFAULT = "8"; /** Name and default value for the tag pair delimiter property. */ public static final String PAIR_DELIMITER_PROPERTY = "tagpairdelimiter"; public static final String PAIR_DELIMITER_PROPERTY_DEFAULT = "="; /** Name and default value for the delete string delimiter property. */ public static final String DELETE_DELIMITER_PROPERTY = "deletedelimiter"; public static final String DELETE_DELIMITER_PROPERTY_DEFAULT = ":"; /** Name and default value for the random timestamp write order property. */ public static final String RANDOMIZE_TIMESTAMP_ORDER_PROPERTY = "randomwritetimestamporder"; public static final String RANDOMIZE_TIMESTAMP_ORDER_PROPERTY_DEFAULT = "false"; /** Name and default value for the random time series write order property. */ public static final String RANDOMIZE_TIMESERIES_ORDER_PROPERTY = "randomtimeseriesorder"; public static final String RANDOMIZE_TIMESERIES_ORDER_PROPERTY_DEFAULT = "true"; /** Name and default value for the value types property. */ public static final String VALUE_TYPE_PROPERTY = "valuetype"; public static final String VALUE_TYPE_PROPERTY_DEFAULT = "floats"; /** Name and default value for the sparsity property. */ public static final String SPARSITY_PROPERTY = "sparsity"; public static final String SPARSITY_PROPERTY_DEFAULT = "0.00"; /** Name and default value for the delayed series percentage property. */ public static final String DELAYED_SERIES_PROPERTY = "delayedseries"; public static final String DELAYED_SERIES_PROPERTY_DEFAULT = "0.10"; /** Name and default value for the delayed series intervals property. */ public static final String DELAYED_INTERVALS_PROPERTY = "delayedintervals"; public static final String DELAYED_INTERVALS_PROPERTY_DEFAULT = "5"; /** Name and default value for the query time span property. */ public static final String QUERY_TIMESPAN_PROPERTY = "querytimespan"; public static final String QUERY_TIMESPAN_PROPERTY_DEFAULT = "0"; /** Name and default value for the randomized query time span property. */ public static final String QUERY_RANDOM_TIMESPAN_PROPERTY = "queryrandomtimespan"; public static final String QUERY_RANDOM_TIMESPAN_PROPERTY_DEFAULT = "false"; /** Name and default value for the query time stamp delimiter property. */ public static final String QUERY_TIMESPAN_DELIMITER_PROPERTY = "querytimespandelimiter"; public static final String QUERY_TIMESPAN_DELIMITER_PROPERTY_DEFAULT = ","; /** Name and default value for the group-by key property. */ public static final String GROUPBY_KEY_PROPERTY = "groupbykey"; public static final String GROUPBY_KEY_PROPERTY_DEFAULT = "YCSBGB"; /** Name and default value for the group-by function property. */ public static final String GROUPBY_PROPERTY = "groupbyfunction"; /** Name and default value for the group-by key map property. */ public static final String GROUPBY_KEYS_PROPERTY = "groupbykeys"; /** Name and default value for the downsampling key property. */ public static final String DOWNSAMPLING_KEY_PROPERTY = "downsamplingkey"; public static final String DOWNSAMPLING_KEY_PROPERTY_DEFAULT = "YCSBDS"; /** Name and default value for the downsampling function property. */ public static final String DOWNSAMPLING_FUNCTION_PROPERTY = "downsamplingfunction"; /** Name and default value for the downsampling interval property. */ public static final String DOWNSAMPLING_INTERVAL_PROPERTY = "downsamplinginterval"; /** The properties to pull settings from. */ protected Properties properties; /** Generators for keys, tag keys and tag values. */ protected Generator keyGenerator; protected Generator tagKeyGenerator; protected Generator tagValueGenerator; /** The timestamp key, defaults to "YCSBTS". */ protected String timestampKey; /** The value key, defaults to "YCSBDS". */ protected String valueKey; /** The number of time units in between timestamps. */ protected int timestampInterval; /** The units of time the timestamp and various intervals represent. */ protected TimeUnit timeUnits; /** Whether or not to randomize the timestamp order when writing. */ protected boolean randomizeTimestampOrder; /** Whether or not to randomize (shuffle) the time series order. NOT compatible * with data integrity. */ protected boolean randomizeTimeseriesOrder; /** The type of values to generate when writing data. */ protected ValueType valueType; /** Used to calculate an offset for each time series. */ protected int[] cumulativeCardinality; /** The calculated total cardinality based on the config. */ protected int totalCardinality; /** The calculated per-time-series-key cardinality. I.e. the number of unique * tag key and value combinations. */ protected int perKeyCardinality; /** How much data to scan for in each call. */ protected NumberGenerator scanlength; /** A generator used to select a random time series key per read/scan. */ protected NumberGenerator keychooser; /** A generator to select what operation to perform during the run phase. */ protected DiscreteGenerator operationchooser; /** The maximum number of interval offsets from the starting timestamp. Calculated * based on the number of records configured for the run. */ protected int maxOffsets; /** The number of records or operations to perform for this run. */ protected int recordcount; /** The number of tag pairs per time series. */ protected int tagPairs; /** The table we'll write to. */ protected String table; /** How many time series keys will be generated. */ protected int numKeys; /** The generated list of possible time series key values. */ protected String[] keys; /** The generated list of possible tag key values. */ protected String[] tagKeys; /** The generated list of possible tag value values. */ protected String[] tagValues; /** The cardinality for each tag key. */ protected int[] tagCardinality; /** A helper to skip non-incrementing tag values. */ protected int firstIncrementableCardinality; /** How sparse the data written should be. */ protected double sparsity; /** The percentage of time series that should be delayed in writes. */ protected double delayedSeries; /** The maximum number of intervals to delay a series. */ protected int delayedIntervals; /** Optional query time interval during reads/scans. */ protected int queryTimeSpan; /** Whether or not the actual interval should be randomly chosen, using * queryTimeSpan as the maximum value. */ protected boolean queryRandomTimeSpan; /** The delimiter for tag pairs in fields. */ protected String tagPairDelimiter; /** The delimiter between parameters for the delete key. */ protected String deleteDelimiter; /** The delimiter between timestamps for query time spans. */ protected String queryTimeSpanDelimiter; /** Whether or not to issue group-by queries. */ protected boolean groupBy; /** The key used for group-by tag keys. */ protected String groupByKey; /** The function used for group-by's. */ protected String groupByFunction; /** The tag keys to group on. */ protected boolean[] groupBys; /** Whether or not to issue downsampling queries. */ protected boolean downsample; /** The key used for downsampling tag keys. */ protected String downsampleKey; /** The downsampling function. */ protected String downsampleFunction; /** The downsampling interval. */ protected int downsampleInterval; /** * Set to true if want to check correctness of reads. Must also * be set to true during loading phase to function. */ protected boolean dataintegrity; /** Measurements to write data integrity results to. */ protected Measurements measurements = Measurements.getMeasurements(); @Override public void init(final Properties p) throws WorkloadException { properties = p; recordcount = Integer.parseInt(p.getProperty(Client.RECORD_COUNT_PROPERTY, Client.DEFAULT_RECORD_COUNT)); if (recordcount == 0) { recordcount = Integer.MAX_VALUE; } timestampKey = p.getProperty(TIMESTAMP_KEY_PROPERTY, TIMESTAMP_KEY_PROPERTY_DEFAULT); valueKey = p.getProperty(VALUE_KEY_PROPERTY, VALUE_KEY_PROPERTY_DEFAULT); operationchooser = CoreWorkload.createOperationGenerator(properties); final int maxscanlength = Integer.parseInt(p.getProperty(CoreWorkload.MAX_SCAN_LENGTH_PROPERTY, CoreWorkload.MAX_SCAN_LENGTH_PROPERTY_DEFAULT)); String scanlengthdistrib = p.getProperty(CoreWorkload.SCAN_LENGTH_DISTRIBUTION_PROPERTY, CoreWorkload.SCAN_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT); if (scanlengthdistrib.compareTo("uniform") == 0) { scanlength = new UniformLongGenerator(1, maxscanlength); } else if (scanlengthdistrib.compareTo("zipfian") == 0) { scanlength = new ZipfianGenerator(1, maxscanlength); } else { throw new WorkloadException( "Distribution \"" + scanlengthdistrib + "\" not allowed for scan length"); } randomizeTimestampOrder = Boolean.parseBoolean(p.getProperty( RANDOMIZE_TIMESTAMP_ORDER_PROPERTY, RANDOMIZE_TIMESTAMP_ORDER_PROPERTY_DEFAULT)); randomizeTimeseriesOrder = Boolean.parseBoolean(p.getProperty( RANDOMIZE_TIMESERIES_ORDER_PROPERTY, RANDOMIZE_TIMESERIES_ORDER_PROPERTY_DEFAULT)); // setup the cardinality numKeys = Integer.parseInt(p.getProperty(CoreWorkload.FIELD_COUNT_PROPERTY, CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT)); tagPairs = Integer.parseInt(p.getProperty(TAG_COUNT_PROPERTY, TAG_COUNT_PROPERTY_DEFAULT)); sparsity = Double.parseDouble(p.getProperty(SPARSITY_PROPERTY, SPARSITY_PROPERTY_DEFAULT)); tagCardinality = new int[tagPairs]; final String requestdistrib = p.getProperty(CoreWorkload.REQUEST_DISTRIBUTION_PROPERTY, CoreWorkload.REQUEST_DISTRIBUTION_PROPERTY_DEFAULT); if (requestdistrib.compareTo("uniform") == 0) { keychooser = new UniformLongGenerator(0, numKeys - 1); } else if (requestdistrib.compareTo("sequential") == 0) { keychooser = new SequentialGenerator(0, numKeys - 1); } else if (requestdistrib.compareTo("zipfian") == 0) { keychooser = new ScrambledZipfianGenerator(0, numKeys - 1); //} else if (requestdistrib.compareTo("latest") == 0) { // keychooser = new SkewedLatestGenerator(transactioninsertkeysequence); } else if (requestdistrib.equals("hotspot")) { double hotsetfraction = Double.parseDouble(p.getProperty(CoreWorkload.HOTSPOT_DATA_FRACTION, CoreWorkload.HOTSPOT_DATA_FRACTION_DEFAULT)); double hotopnfraction = Double.parseDouble(p.getProperty(CoreWorkload.HOTSPOT_OPN_FRACTION, CoreWorkload.HOTSPOT_OPN_FRACTION_DEFAULT)); keychooser = new HotspotIntegerGenerator(0, numKeys - 1, hotsetfraction, hotopnfraction); } else { throw new WorkloadException("Unknown request distribution \"" + requestdistrib + "\""); } // figure out the start timestamp based on the units, cardinality and interval try { timestampInterval = Integer.parseInt(p.getProperty( TIMESTAMP_INTERVAL_PROPERTY, TIMESTAMP_INTERVAL_PROPERTY_DEFAULT)); } catch (NumberFormatException nfe) { throw new WorkloadException("Unable to parse the " + TIMESTAMP_INTERVAL_PROPERTY, nfe); } try { timeUnits = TimeUnit.valueOf(p.getProperty(TIMESTAMP_UNITS_PROPERTY, TIMESTAMP_UNITS_PROPERTY_DEFAULT).toUpperCase()); } catch (IllegalArgumentException e) { throw new WorkloadException("Unknown time unit type", e); } if (timeUnits == TimeUnit.NANOSECONDS || timeUnits == TimeUnit.MICROSECONDS) { throw new WorkloadException("YCSB doesn't support " + timeUnits + " at this time."); } tagPairDelimiter = p.getProperty(PAIR_DELIMITER_PROPERTY, PAIR_DELIMITER_PROPERTY_DEFAULT); deleteDelimiter = p.getProperty(DELETE_DELIMITER_PROPERTY, DELETE_DELIMITER_PROPERTY_DEFAULT); dataintegrity = Boolean.parseBoolean( p.getProperty(CoreWorkload.DATA_INTEGRITY_PROPERTY, CoreWorkload.DATA_INTEGRITY_PROPERTY_DEFAULT)); queryTimeSpan = Integer.parseInt(p.getProperty(QUERY_TIMESPAN_PROPERTY, QUERY_TIMESPAN_PROPERTY_DEFAULT)); queryRandomTimeSpan = Boolean.parseBoolean(p.getProperty(QUERY_RANDOM_TIMESPAN_PROPERTY, QUERY_RANDOM_TIMESPAN_PROPERTY_DEFAULT)); queryTimeSpanDelimiter = p.getProperty(QUERY_TIMESPAN_DELIMITER_PROPERTY, QUERY_TIMESPAN_DELIMITER_PROPERTY_DEFAULT); groupByKey = p.getProperty(GROUPBY_KEY_PROPERTY, GROUPBY_KEY_PROPERTY_DEFAULT); groupByFunction = p.getProperty(GROUPBY_PROPERTY); if (groupByFunction != null && !groupByFunction.isEmpty()) { final String groupByKeys = p.getProperty(GROUPBY_KEYS_PROPERTY); if (groupByKeys == null || groupByKeys.isEmpty()) { throw new WorkloadException("Group by was enabled but no keys were specified."); } final String[] gbKeys = groupByKeys.split(","); if (gbKeys.length != tagKeys.length) { throw new WorkloadException("Only " + gbKeys.length + " group by keys " + "were specified but there were " + tagKeys.length + " tag keys given."); } groupBys = new boolean[gbKeys.length]; for (int i = 0; i < gbKeys.length; i++) { groupBys[i] = Integer.parseInt(gbKeys[i].trim()) == 0 ? false : true; } groupBy = true; } downsampleKey = p.getProperty(DOWNSAMPLING_KEY_PROPERTY, DOWNSAMPLING_KEY_PROPERTY_DEFAULT); downsampleFunction = p.getProperty(DOWNSAMPLING_FUNCTION_PROPERTY); if (downsampleFunction != null && !downsampleFunction.isEmpty()) { final String interval = p.getProperty(DOWNSAMPLING_INTERVAL_PROPERTY); if (interval == null || interval.isEmpty()) { throw new WorkloadException("'" + DOWNSAMPLING_INTERVAL_PROPERTY + "' was missing despite '" + DOWNSAMPLING_FUNCTION_PROPERTY + "' being set."); } downsampleInterval = Integer.parseInt(interval); downsample = true; } delayedSeries = Double.parseDouble(p.getProperty(DELAYED_SERIES_PROPERTY, DELAYED_SERIES_PROPERTY_DEFAULT)); delayedIntervals = Integer.parseInt(p.getProperty(DELAYED_INTERVALS_PROPERTY, DELAYED_INTERVALS_PROPERTY_DEFAULT)); valueType = ValueType.fromString(p.getProperty(VALUE_TYPE_PROPERTY, VALUE_TYPE_PROPERTY_DEFAULT)); table = p.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT); initKeysAndTags(); validateSettings(); } @Override public Object initThread(Properties p, int mythreadid, int threadcount) throws WorkloadException { if (properties == null) { throw new WorkloadException("Workload has not been initialized."); } return new ThreadState(mythreadid, threadcount); } @Override public boolean doInsert(DB db, Object threadstate) { if (threadstate == null) { throw new IllegalStateException("Missing thread state."); } final Map tags = new TreeMap(); final String key = ((ThreadState)threadstate).nextDataPoint(tags, true); if (db.insert(table, key, tags) == Status.OK) { return true; } return false; } @Override public boolean doTransaction(DB db, Object threadstate) { if (threadstate == null) { throw new IllegalStateException("Missing thread state."); } switch (operationchooser.nextString()) { case "READ": doTransactionRead(db, threadstate); break; case "UPDATE": doTransactionUpdate(db, threadstate); break; case "INSERT": doTransactionInsert(db, threadstate); break; case "SCAN": doTransactionScan(db, threadstate); break; case "DELETE": doTransactionDelete(db, threadstate); break; default: return false; } return true; } protected void doTransactionRead(final DB db, Object threadstate) { final ThreadState state = (ThreadState) threadstate; final String keyname = keys[keychooser.nextValue().intValue()]; int offsets = state.queryOffsetGenerator.nextValue().intValue(); //int offsets = Utils.random().nextInt(maxOffsets - 1); final long startTimestamp; if (offsets > 0) { startTimestamp = state.startTimestamp + state.timestampGenerator.getOffset(offsets); } else { startTimestamp = state.startTimestamp; } // rando tags Set fields = new HashSet(); for (int i = 0; i < tagPairs; ++i) { if (groupBy && groupBys[i]) { fields.add(tagKeys[i]); } else { fields.add(tagKeys[i] + tagPairDelimiter + tagValues[Utils.random().nextInt(tagCardinality[i])]); } } if (queryTimeSpan > 0) { final long endTimestamp; if (queryRandomTimeSpan) { endTimestamp = startTimestamp + (timestampInterval * Utils.random().nextInt(queryTimeSpan / timestampInterval)); } else { endTimestamp = startTimestamp + queryTimeSpan; } fields.add(timestampKey + tagPairDelimiter + startTimestamp + queryTimeSpanDelimiter + endTimestamp); } else { fields.add(timestampKey + tagPairDelimiter + startTimestamp); } if (groupBy) { fields.add(groupByKey + tagPairDelimiter + groupByFunction); } if (downsample) { fields.add(downsampleKey + tagPairDelimiter + downsampleFunction + downsampleInterval); } final Map cells = new HashMap(); final Status status = db.read(table, keyname, fields, cells); if (dataintegrity && status == Status.OK) { verifyRow(keyname, cells); } } protected void doTransactionUpdate(final DB db, Object threadstate) { if (threadstate == null) { throw new IllegalStateException("Missing thread state."); } final Map tags = new TreeMap(); final String key = ((ThreadState)threadstate).nextDataPoint(tags, false); db.update(table, key, tags); } protected void doTransactionInsert(final DB db, Object threadstate) { doInsert(db, threadstate); } protected void doTransactionScan(final DB db, Object threadstate) { final ThreadState state = (ThreadState) threadstate; final String keyname = keys[Utils.random().nextInt(keys.length)]; // choose a random scan length int len = scanlength.nextValue().intValue(); int offsets = Utils.random().nextInt(maxOffsets - 1); final long startTimestamp; if (offsets > 0) { startTimestamp = state.startTimestamp + state.timestampGenerator.getOffset(offsets); } else { startTimestamp = state.startTimestamp; } // rando tags Set fields = new HashSet(); for (int i = 0; i < tagPairs; ++i) { if (groupBy && groupBys[i]) { fields.add(tagKeys[i]); } else { fields.add(tagKeys[i] + tagPairDelimiter + tagValues[Utils.random().nextInt(tagCardinality[i])]); } } if (queryTimeSpan > 0) { final long endTimestamp; if (queryRandomTimeSpan) { endTimestamp = startTimestamp + (timestampInterval * Utils.random().nextInt(queryTimeSpan / timestampInterval)); } else { endTimestamp = startTimestamp + queryTimeSpan; } fields.add(timestampKey + tagPairDelimiter + startTimestamp + queryTimeSpanDelimiter + endTimestamp); } else { fields.add(timestampKey + tagPairDelimiter + startTimestamp); } if (groupBy) { fields.add(groupByKey + tagPairDelimiter + groupByFunction); } if (downsample) { fields.add(downsampleKey + tagPairDelimiter + downsampleFunction + tagPairDelimiter + downsampleInterval); } final Vector> results = new Vector>(); db.scan(table, keyname, len, fields, results); } protected void doTransactionDelete(final DB db, Object threadstate) { final ThreadState state = (ThreadState) threadstate; final StringBuilder buf = new StringBuilder().append(keys[Utils.random().nextInt(keys.length)]); int offsets = Utils.random().nextInt(maxOffsets - 1); final long startTimestamp; if (offsets > 0) { startTimestamp = state.startTimestamp + state.timestampGenerator.getOffset(offsets); } else { startTimestamp = state.startTimestamp; } // rando tags for (int i = 0; i < tagPairs; ++i) { if (groupBy && groupBys[i]) { buf.append(deleteDelimiter) .append(tagKeys[i]); } else { buf.append(deleteDelimiter).append(tagKeys[i] + tagPairDelimiter + tagValues[Utils.random().nextInt(tagCardinality[i])]); } } if (queryTimeSpan > 0) { final long endTimestamp; if (queryRandomTimeSpan) { endTimestamp = startTimestamp + (timestampInterval * Utils.random().nextInt(queryTimeSpan / timestampInterval)); } else { endTimestamp = startTimestamp + queryTimeSpan; } buf.append(deleteDelimiter) .append(timestampKey + tagPairDelimiter + startTimestamp + queryTimeSpanDelimiter + endTimestamp); } else { buf.append(deleteDelimiter) .append(timestampKey + tagPairDelimiter + startTimestamp); } db.delete(table, buf.toString()); } /** * Parses the values returned by a read or scan operation and determines whether * or not the integer value matches the hash and timestamp of the original timestamp. * Only works for raw data points, will not work for group-by's or downsampled data. * @param key The time series key. * @param cells The cells read by the DB. * @return {@link Status#OK} if the data matched or {@link Status#UNEXPECTED_STATE} if * the data did not match. */ protected Status verifyRow(final String key, final Map cells) { Status verifyStatus = Status.UNEXPECTED_STATE; long startTime = System.nanoTime(); double value = 0; long timestamp = 0; final TreeMap validationTags = new TreeMap(); for (final Entry entry : cells.entrySet()) { if (entry.getKey().equals(timestampKey)) { final NumericByteIterator it = (NumericByteIterator) entry.getValue(); timestamp = it.getLong(); } else if (entry.getKey().equals(valueKey)) { final NumericByteIterator it = (NumericByteIterator) entry.getValue(); value = it.isFloatingPoint() ? it.getDouble() : it.getLong(); } else { validationTags.put(entry.getKey(), entry.getValue().toString()); } } if (validationFunction(key, timestamp, validationTags) == value) { verifyStatus = Status.OK; } long endTime = System.nanoTime(); measurements.measure("VERIFY", (int) (endTime - startTime) / 1000); measurements.reportStatus("VERIFY", verifyStatus); return verifyStatus; } /** * Function used for generating a deterministic hash based on the combination * of metric, tags and timestamp. * @param key A non-null string representing the key. * @param timestamp A timestamp in the proper units for the workload. * @param tags A non-null map of tag keys and values NOT including the YCSB * key or timestamp. * @return A hash value as an 8 byte integer. */ protected long validationFunction(final String key, final long timestamp, final TreeMap tags) { final StringBuilder validationBuffer = new StringBuilder(keys[0].length() + (tagPairs * tagKeys[0].length()) + (tagPairs * tagCardinality[1])); for (final Entry pair : tags.entrySet()) { validationBuffer.append(pair.getKey()).append(pair.getValue()); } return (long) validationBuffer.toString().hashCode() ^ timestamp; } /** * Breaks out the keys, tags and cardinality initialization in another method * to keep CheckStyle happy. * @throws WorkloadException If something goes pear shaped. */ protected void initKeysAndTags() throws WorkloadException { final int keyLength = Integer.parseInt(properties.getProperty( CoreWorkload.FIELD_LENGTH_PROPERTY, CoreWorkload.FIELD_LENGTH_PROPERTY_DEFAULT)); final int tagKeyLength = Integer.parseInt(properties.getProperty( TAG_KEY_LENGTH_PROPERTY, TAG_KEY_LENGTH_PROPERTY_DEFAULT)); final int tagValueLength = Integer.parseInt(properties.getProperty( TAG_VALUE_LENGTH_PROPERTY, TAG_VALUE_LENGTH_PROPERTY_DEFAULT)); keyGenerator = new IncrementingPrintableStringGenerator(keyLength); tagKeyGenerator = new IncrementingPrintableStringGenerator(tagKeyLength); tagValueGenerator = new IncrementingPrintableStringGenerator(tagValueLength); final int threads = Integer.parseInt(properties.getProperty(Client.THREAD_COUNT_PROPERTY, "1")); final String tagCardinalityString = properties.getProperty( TAG_CARDINALITY_PROPERTY, TAG_CARDINALITY_PROPERTY_DEFAULT); final String[] tagCardinalityParts = tagCardinalityString.split(","); int idx = 0; totalCardinality = numKeys; perKeyCardinality = 1; int maxCardinality = 0; for (final String card : tagCardinalityParts) { try { tagCardinality[idx] = Integer.parseInt(card.trim()); } catch (NumberFormatException nfe) { throw new WorkloadException("Unable to parse cardinality: " + card, nfe); } if (tagCardinality[idx] < 1) { throw new WorkloadException("Cardinality must be greater than zero: " + tagCardinality[idx]); } totalCardinality *= tagCardinality[idx]; perKeyCardinality *= tagCardinality[idx]; if (tagCardinality[idx] > maxCardinality) { maxCardinality = tagCardinality[idx]; } ++idx; if (idx >= tagPairs) { // we have more cardinalities than tag keys so bail at this point. break; } } if (numKeys < threads) { throw new WorkloadException("Field count " + numKeys + " (keys for time " + "series workloads) must be greater or equal to the number of " + "threads " + threads); } // fill tags without explicit cardinality with 1 if (idx < tagPairs) { tagCardinality[idx++] = 1; } for (int i = 0; i < tagCardinality.length; ++i) { if (tagCardinality[i] > 1) { firstIncrementableCardinality = i; break; } } keys = new String[numKeys]; tagKeys = new String[tagPairs]; tagValues = new String[maxCardinality]; for (int i = 0; i < numKeys; ++i) { keys[i] = keyGenerator.nextString(); } for (int i = 0; i < tagPairs; ++i) { tagKeys[i] = tagKeyGenerator.nextString(); } for (int i = 0; i < maxCardinality; i++) { tagValues[i] = tagValueGenerator.nextString(); } if (randomizeTimeseriesOrder) { Utils.shuffleArray(keys); Utils.shuffleArray(tagValues); } maxOffsets = (recordcount / totalCardinality) + 1; final int[] keyAndTagCardinality = new int[tagPairs + 1]; keyAndTagCardinality[0] = numKeys; for (int i = 0; i < tagPairs; i++) { keyAndTagCardinality[i + 1] = tagCardinality[i]; } cumulativeCardinality = new int[keyAndTagCardinality.length]; for (int i = 0; i < keyAndTagCardinality.length; i++) { int cumulation = 1; for (int x = i; x <= keyAndTagCardinality.length - 1; x++) { cumulation *= keyAndTagCardinality[x]; } if (i > 0) { cumulativeCardinality[i - 1] = cumulation; } } cumulativeCardinality[cumulativeCardinality.length - 1] = 1; } /** * Makes sure the settings as given are compatible. * @throws WorkloadException If one or more settings were invalid. */ protected void validateSettings() throws WorkloadException { if (dataintegrity) { if (valueType != ValueType.INTEGERS) { throw new WorkloadException("Data integrity was enabled. 'valuetype' must " + "be set to 'integers'."); } if (groupBy) { throw new WorkloadException("Data integrity was enabled. 'groupbyfunction' must " + "be empty or null."); } if (downsample) { throw new WorkloadException("Data integrity was enabled. 'downsamplingfunction' must " + "be empty or null."); } if (queryTimeSpan > 0) { throw new WorkloadException("Data integrity was enabled. 'querytimespan' must " + "be empty or 0."); } if (randomizeTimeseriesOrder) { throw new WorkloadException("Data integrity was enabled. 'randomizetimeseriesorder' must " + "be false."); } final String startTimestamp = properties.getProperty(CoreWorkload.INSERT_START_PROPERTY); if (startTimestamp == null || startTimestamp.isEmpty()) { throw new WorkloadException("Data integrity was enabled. 'insertstart' must " + "be set to a Unix Epoch timestamp."); } } } /** * Thread state class holding thread local generators and indices. */ protected class ThreadState { /** The timestamp generator for this thread. */ protected final UnixEpochTimestampGenerator timestampGenerator; /** An offset generator to select a random offset for queries. */ protected final NumberGenerator queryOffsetGenerator; /** The current write key index. */ protected int keyIdx; /** The starting fence for writing keys. */ protected int keyIdxStart; /** The ending fence for writing keys. */ protected int keyIdxEnd; /** Indices for each tag value for writes. */ protected int[] tagValueIdxs; /** Whether or not all time series have written values for the current timestamp. */ protected boolean rollover; /** The starting timestamp. */ protected long startTimestamp; /** * Default ctor. * @param threadID The zero based thread ID. * @param threadCount The total number of threads. * @throws WorkloadException If something went pear shaped. */ protected ThreadState(final int threadID, final int threadCount) throws WorkloadException { int totalThreads = threadCount > 0 ? threadCount : 1; if (threadID >= totalThreads) { throw new IllegalStateException("Thread ID " + threadID + " cannot be greater " + "than or equal than the thread count " + totalThreads); } if (keys.length < threadCount) { throw new WorkloadException("Thread count " + totalThreads + " must be greater " + "than or equal to key count " + keys.length); } int keysPerThread = keys.length / totalThreads; keyIdx = keysPerThread * threadID; keyIdxStart = keyIdx; if (totalThreads - 1 == threadID) { keyIdxEnd = keys.length; } else { keyIdxEnd = keyIdxStart + keysPerThread; } tagValueIdxs = new int[tagPairs]; // all zeros final String startingTimestamp = properties.getProperty(CoreWorkload.INSERT_START_PROPERTY); if (startingTimestamp == null || startingTimestamp.isEmpty()) { timestampGenerator = randomizeTimestampOrder ? new RandomDiscreteTimestampGenerator(timestampInterval, timeUnits, maxOffsets) : new UnixEpochTimestampGenerator(timestampInterval, timeUnits); } else { try { timestampGenerator = randomizeTimestampOrder ? new RandomDiscreteTimestampGenerator(timestampInterval, timeUnits, Long.parseLong(startingTimestamp), maxOffsets) : new UnixEpochTimestampGenerator(timestampInterval, timeUnits, Long.parseLong(startingTimestamp)); } catch (NumberFormatException nfe) { throw new WorkloadException("Unable to parse the " + CoreWorkload.INSERT_START_PROPERTY, nfe); } } // Set the last value properly for the timestamp, otherwise it may start // one interval ago. startTimestamp = timestampGenerator.nextValue(); // TODO - pick it queryOffsetGenerator = new UniformLongGenerator(0, maxOffsets - 2); } /** * Generates the next write value for thread. * @param map An initialized map to populate with tag keys and values as well * as the timestamp and actual value. * @param isInsert Whether or not it's an insert or an update. Updates will pick * an older timestamp (if random isn't enabled). * @return The next key to write. */ protected String nextDataPoint(final Map map, final boolean isInsert) { int iterations = sparsity <= 0 ? 1 : Utils.random().nextInt((int) ((double) perKeyCardinality * sparsity)); if (iterations < 1) { iterations = 1; } while (true) { iterations--; if (rollover) { timestampGenerator.nextValue(); rollover = false; } String key = null; if (iterations <= 0) { final TreeMap validationTags; if (dataintegrity) { validationTags = new TreeMap(); } else { validationTags = null; } key = keys[keyIdx]; int overallIdx = keyIdx * cumulativeCardinality[0]; for (int i = 0; i < tagPairs; ++i) { int tvidx = tagValueIdxs[i]; map.put(tagKeys[i], new StringByteIterator(tagValues[tvidx])); if (dataintegrity) { validationTags.put(tagKeys[i], tagValues[tvidx]); } if (delayedSeries > 0) { overallIdx += (tvidx * cumulativeCardinality[i + 1]); } } if (!isInsert) { final long delta = (timestampGenerator.currentValue() - startTimestamp) / timestampInterval; final int intervals = Utils.random().nextInt((int) delta); map.put(timestampKey, new NumericByteIterator(startTimestamp + (intervals * timestampInterval))); } else if (delayedSeries > 0) { // See if the series falls in a delay bucket and calculate an offset earlier // than the current timestamp value if so. double pct = (double) overallIdx / (double) totalCardinality; if (pct < delayedSeries) { int modulo = overallIdx % delayedIntervals; if (modulo < 0) { modulo *= -1; } map.put(timestampKey, new NumericByteIterator(timestampGenerator.currentValue() - timestampInterval * modulo)); } else { map.put(timestampKey, new NumericByteIterator(timestampGenerator.currentValue())); } } else { map.put(timestampKey, new NumericByteIterator(timestampGenerator.currentValue())); } if (dataintegrity) { map.put(valueKey, new NumericByteIterator(validationFunction(key, timestampGenerator.currentValue(), validationTags))); } else { switch (valueType) { case INTEGERS: map.put(valueKey, new NumericByteIterator(Utils.random().nextInt())); break; case FLOATS: map.put(valueKey, new NumericByteIterator( Utils.random().nextDouble() * (double) 100000)); break; case MIXED: if (Utils.random().nextBoolean()) { map.put(valueKey, new NumericByteIterator(Utils.random().nextInt())); } else { map.put(valueKey, new NumericByteIterator( Utils.random().nextDouble() * (double) 100000)); } break; default: throw new IllegalStateException("Somehow we didn't have a value " + "type configured that we support: " + valueType); } } } boolean tagRollover = false; for (int i = tagCardinality.length - 1; i >= 0; --i) { if (tagCardinality[i] <= 1) { // nothing to increment here continue; } if (tagValueIdxs[i] + 1 >= tagCardinality[i]) { tagValueIdxs[i] = 0; if (i == firstIncrementableCardinality) { tagRollover = true; } } else { ++tagValueIdxs[i]; break; } } if (tagRollover) { if (keyIdx + 1 >= keyIdxEnd) { keyIdx = keyIdxStart; rollover = true; } else { ++keyIdx; } } if (iterations <= 0) { return key; } } } } } ================================================ FILE: core/src/main/java/com/yahoo/ycsb/workloads/package-info.java ================================================ /* * Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB workloads. */ package com.yahoo.ycsb.workloads; ================================================ FILE: core/src/main/resources/project.properties ================================================ version=${project.version} ================================================ FILE: core/src/test/java/com/yahoo/ycsb/TestByteIterator.java ================================================ /** * Copyright (c) 2012 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import org.testng.annotations.Test; import static org.testng.AssertJUnit.*; public class TestByteIterator { @Test public void testRandomByteIterator() { int size = 100; ByteIterator itor = new RandomByteIterator(size); assertTrue(itor.hasNext()); assertEquals(size, itor.bytesLeft()); assertEquals(size, itor.toString().getBytes().length); assertFalse(itor.hasNext()); assertEquals(0, itor.bytesLeft()); itor = new RandomByteIterator(size); assertEquals(size, itor.toArray().length); assertFalse(itor.hasNext()); assertEquals(0, itor.bytesLeft()); } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/TestNumericByteIterator.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import org.testng.annotations.Test; import static org.testng.AssertJUnit.*; public class TestNumericByteIterator { @Test public void testLong() throws Exception { NumericByteIterator it = new NumericByteIterator(42L); assertFalse(it.isFloatingPoint()); assertEquals(42L, it.getLong()); try { it.getDouble(); fail("Expected IllegalStateException."); } catch (IllegalStateException e) { } try { it.next(); fail("Expected UnsupportedOperationException."); } catch (UnsupportedOperationException e) { } assertEquals(8, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(7, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(6, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(5, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(4, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(3, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(2, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(1, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 42, (byte) it.nextByte()); assertEquals(0, it.bytesLeft()); assertFalse(it.hasNext()); it.reset(); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); } @Test public void testDouble() throws Exception { NumericByteIterator it = new NumericByteIterator(42.75); assertTrue(it.isFloatingPoint()); assertEquals(42.75, it.getDouble(), 0.001); try { it.getLong(); fail("Expected IllegalStateException."); } catch (IllegalStateException e) { } try { it.next(); fail("Expected UnsupportedOperationException."); } catch (UnsupportedOperationException e) { } assertEquals(8, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 64, (byte) it.nextByte()); assertEquals(7, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 69, (byte) it.nextByte()); assertEquals(6, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 96, (byte) it.nextByte()); assertEquals(5, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(4, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(3, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(2, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(1, it.bytesLeft()); assertTrue(it.hasNext()); assertEquals((byte) 0, (byte) it.nextByte()); assertEquals(0, it.bytesLeft()); assertFalse(it.hasNext()); it.reset(); assertTrue(it.hasNext()); assertEquals((byte) 64, (byte) it.nextByte()); } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/TestStatus.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import org.testng.annotations.Test; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; /** * Test class for {@link Status}. */ public class TestStatus { @Test public void testAcceptableStatus() { assertTrue(Status.OK.isOk()); assertTrue(Status.BATCHED_OK.isOk()); assertFalse(Status.BAD_REQUEST.isOk()); assertFalse(Status.ERROR.isOk()); assertFalse(Status.FORBIDDEN.isOk()); assertFalse(Status.NOT_FOUND.isOk()); assertFalse(Status.NOT_IMPLEMENTED.isOk()); assertFalse(Status.SERVICE_UNAVAILABLE.isOk()); assertFalse(Status.UNEXPECTED_STATE.isOk()); } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/TestUtils.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.util.Arrays; import org.testng.annotations.Test; public class TestUtils { @Test public void bytesToFromLong() throws Exception { byte[] bytes = new byte[8]; assertEquals(Utils.bytesToLong(bytes), 0L); assertArrayEquals(Utils.longToBytes(0), bytes); bytes[7] = 1; assertEquals(Utils.bytesToLong(bytes), 1L); assertArrayEquals(Utils.longToBytes(1L), bytes); bytes = new byte[] { 127, -1, -1, -1, -1, -1, -1, -1 }; assertEquals(Utils.bytesToLong(bytes), Long.MAX_VALUE); assertArrayEquals(Utils.longToBytes(Long.MAX_VALUE), bytes); bytes = new byte[] { -128, 0, 0, 0, 0, 0, 0, 0 }; assertEquals(Utils.bytesToLong(bytes), Long.MIN_VALUE); assertArrayEquals(Utils.longToBytes(Long.MIN_VALUE), bytes); bytes = new byte[] { (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF }; assertEquals(Utils.bytesToLong(bytes), -1L); assertArrayEquals(Utils.longToBytes(-1L), bytes); // if the array is too long we just skip the remainder bytes = new byte[] { 0, 0, 0, 0, 0, 0, 0, 1, 42, 42, 42 }; assertEquals(Utils.bytesToLong(bytes), 1L); } @Test public void bytesToFromDouble() throws Exception { byte[] bytes = new byte[8]; assertEquals(Utils.bytesToDouble(bytes), 0, 0.0001); assertArrayEquals(Utils.doubleToBytes(0), bytes); bytes = new byte[] { 63, -16, 0, 0, 0, 0, 0, 0 }; assertEquals(Utils.bytesToDouble(bytes), 1, 0.0001); assertArrayEquals(Utils.doubleToBytes(1), bytes); bytes = new byte[] { -65, -16, 0, 0, 0, 0, 0, 0 }; assertEquals(Utils.bytesToDouble(bytes), -1, 0.0001); assertArrayEquals(Utils.doubleToBytes(-1), bytes); bytes = new byte[] { 127, -17, -1, -1, -1, -1, -1, -1 }; assertEquals(Utils.bytesToDouble(bytes), Double.MAX_VALUE, 0.0001); assertArrayEquals(Utils.doubleToBytes(Double.MAX_VALUE), bytes); bytes = new byte[] { 0, 0, 0, 0, 0, 0, 0, 1 }; assertEquals(Utils.bytesToDouble(bytes), Double.MIN_VALUE, 0.0001); assertArrayEquals(Utils.doubleToBytes(Double.MIN_VALUE), bytes); bytes = new byte[] { 127, -8, 0, 0, 0, 0, 0, 0 }; assertTrue(Double.isNaN(Utils.bytesToDouble(bytes))); assertArrayEquals(Utils.doubleToBytes(Double.NaN), bytes); bytes = new byte[] { 63, -16, 0, 0, 0, 0, 0, 0, 42, 42, 42 }; assertEquals(Utils.bytesToDouble(bytes), 1, 0.0001); } @Test (expectedExceptions = NullPointerException.class) public void bytesToLongNull() throws Exception { Utils.bytesToLong(null); } @Test (expectedExceptions = IndexOutOfBoundsException.class) public void bytesToLongTooShort() throws Exception { Utils.bytesToLong(new byte[] { 0, 0, 0, 0, 0, 0, 0 }); } @Test (expectedExceptions = IllegalArgumentException.class) public void bytesToDoubleTooShort() throws Exception { Utils.bytesToDouble(new byte[] { 0, 0, 0, 0, 0, 0, 0 }); } @Test public void jvmUtils() throws Exception { // This should ALWAYS return at least one thread. assertTrue(Utils.getActiveThreadCount() > 0); // This should always be greater than 0 or something is goofed up in the JVM. assertTrue(Utils.getUsedMemoryBytes() > 0); // Some operating systems may not implement this so we don't have a good // test. Just make sure it doesn't throw an exception. Utils.getSystemLoadAverage(); // This will probably be zero but should never be negative. assertTrue(Utils.getGCTotalCollectionCount() >= 0); // Could be zero similar to GC total collection count assertTrue(Utils.getGCTotalTime() >= 0); // Could be empty assertTrue(Utils.getGCStatst().size() >= 0); } /** * Since this version of TestNG doesn't appear to have an assertArrayEquals, * this will compare the two to make sure they're the same. * @param actual Actual array to validate * @param expected What the array should contain * @throws AssertionError if the test fails. */ public void assertArrayEquals(final byte[] actual, final byte[] expected) { if (actual == null && expected != null) { throw new AssertionError("Expected " + Arrays.toString(expected) + " but found [null]"); } if (actual != null && expected == null) { throw new AssertionError("Expected [null] but found " + Arrays.toString(actual)); } if (actual.length != expected.length) { throw new AssertionError("Expected length " + expected.length + " but found " + actual.length); } for (int i = 0; i < expected.length; i++) { if (actual[i] != expected[i]) { throw new AssertionError("Expected byte [" + expected[i] + "] at index " + i + " but found [" + actual[i] + "]"); } } } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/generator/AcknowledgedCounterGeneratorTest.java ================================================ /** * Copyright (c) 2015-2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import java.util.Random; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import org.testng.annotations.Test; /** * Tests for the AcknowledgedCounterGenerator class. */ public class AcknowledgedCounterGeneratorTest { /** * Test that advancing past {@link Integer#MAX_VALUE} works. */ @Test public void testIncrementPastIntegerMaxValue() { final long toTry = AcknowledgedCounterGenerator.WINDOW_SIZE * 3; AcknowledgedCounterGenerator generator = new AcknowledgedCounterGenerator(Integer.MAX_VALUE - 1000); Random rand = new Random(System.currentTimeMillis()); BlockingQueue pending = new ArrayBlockingQueue(1000); for (long i = 0; i < toTry; ++i) { long value = generator.nextValue(); while (!pending.offer(value)) { Long first = pending.poll(); // Don't always advance by one. if (rand.nextBoolean()) { generator.acknowledge(first); } else { Long second = pending.poll(); pending.add(first); generator.acknowledge(second); } } } } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/generator/TestIncrementingPrintableStringGenerator.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.fail; import java.util.NoSuchElementException; import org.testng.annotations.Test; public class TestIncrementingPrintableStringGenerator { private final static int[] ATOC = new int[] { 65, 66, 67 }; @Test public void rolloverOK() throws Exception { final IncrementingPrintableStringGenerator gen = new IncrementingPrintableStringGenerator(2, ATOC); assertNull(gen.lastValue()); assertEquals(gen.nextValue(), "AA"); assertEquals(gen.lastValue(), "AA"); assertEquals(gen.nextValue(), "AB"); assertEquals(gen.lastValue(), "AB"); assertEquals(gen.nextValue(), "AC"); assertEquals(gen.lastValue(), "AC"); assertEquals(gen.nextValue(), "BA"); assertEquals(gen.lastValue(), "BA"); assertEquals(gen.nextValue(), "BB"); assertEquals(gen.lastValue(), "BB"); assertEquals(gen.nextValue(), "BC"); assertEquals(gen.lastValue(), "BC"); assertEquals(gen.nextValue(), "CA"); assertEquals(gen.lastValue(), "CA"); assertEquals(gen.nextValue(), "CB"); assertEquals(gen.lastValue(), "CB"); assertEquals(gen.nextValue(), "CC"); assertEquals(gen.lastValue(), "CC"); assertEquals(gen.nextValue(), "AA"); // <-- rollover assertEquals(gen.lastValue(), "AA"); } @Test public void rolloverOneCharacterOK() throws Exception { // It would be silly to create a generator with one character. final IncrementingPrintableStringGenerator gen = new IncrementingPrintableStringGenerator(2, new int[] { 65 }); for (int i = 0; i < 5; i++) { assertEquals(gen.nextValue(), "AA"); } } @Test public void rolloverException() throws Exception { final IncrementingPrintableStringGenerator gen = new IncrementingPrintableStringGenerator(2, ATOC); gen.setThrowExceptionOnRollover(true); int i = 0; try { while(i < 11) { ++i; gen.nextValue(); } fail("Expected NoSuchElementException"); } catch (NoSuchElementException e) { assertEquals(i, 10); } } @Test public void rolloverOneCharacterException() throws Exception { // It would be silly to create a generator with one character. final IncrementingPrintableStringGenerator gen = new IncrementingPrintableStringGenerator(2, new int[] { 65 }); gen.setThrowExceptionOnRollover(true); int i = 0; try { while(i < 3) { ++i; gen.nextValue(); } fail("Expected NoSuchElementException"); } catch (NoSuchElementException e) { assertEquals(i, 2); } } @Test public void invalidLengths() throws Exception { try { new IncrementingPrintableStringGenerator(0, ATOC); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { } try { new IncrementingPrintableStringGenerator(-42, ATOC); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { } } @Test public void invalidCharacterSets() throws Exception { try { new IncrementingPrintableStringGenerator(2, null); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { } try { new IncrementingPrintableStringGenerator(2, new int[] {}); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { } } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/generator/TestRandomDiscreteTimestampGenerator.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import org.testng.annotations.Test; import org.testng.collections.Lists; public class TestRandomDiscreteTimestampGenerator { @Test public void systemTime() throws Exception { final RandomDiscreteTimestampGenerator generator = new RandomDiscreteTimestampGenerator(60, TimeUnit.SECONDS, 60); List generated = Lists.newArrayList(); for (int i = 0; i < 60; i++) { generated.add(generator.nextValue()); } assertEquals(generated.size(), 60); try { generator.nextValue(); fail("Expected IllegalStateException"); } catch (IllegalStateException e) { } } @Test public void withStartTime() throws Exception { final RandomDiscreteTimestampGenerator generator = new RandomDiscreteTimestampGenerator(60, TimeUnit.SECONDS, 1072915200L, 60); List generated = Lists.newArrayList(); for (int i = 0; i < 60; i++) { generated.add(generator.nextValue()); } assertEquals(generated.size(), 60); Collections.sort(generated); long ts = 1072915200L - 60; // starts 1 interval in the past for (final long t : generated) { assertEquals(t, ts); ts += 60; } try { generator.nextValue(); fail("Expected IllegalStateException"); } catch (IllegalStateException e) { } } @Test (expectedExceptions = IllegalArgumentException.class) public void tooLarge() throws Exception { new RandomDiscreteTimestampGenerator(60, TimeUnit.SECONDS, RandomDiscreteTimestampGenerator.MAX_INTERVALS + 1); } //TODO - With PowerMockito we could UT the initializeTimestamp(long) call. // Otherwise it would involve creating more functions and that would get ugly. } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/generator/TestUnixEpochTimestampGenerator.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import static org.testng.Assert.assertEquals; import java.util.concurrent.TimeUnit; import org.testng.annotations.Test; public class TestUnixEpochTimestampGenerator { @Test public void defaultCtor() throws Exception { final UnixEpochTimestampGenerator generator = new UnixEpochTimestampGenerator(); final long startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + 60); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + 120); assertEquals((long) generator.lastValue(), startTime + 60); assertEquals((long) generator.nextValue(), startTime + 180); } @Test public void ctorWithIntervalAndUnits() throws Exception { final UnixEpochTimestampGenerator generator = new UnixEpochTimestampGenerator(120, TimeUnit.SECONDS); final long startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + 120); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + 240); assertEquals((long) generator.lastValue(), startTime + 120); } @Test public void ctorWithIntervalAndUnitsAndStart() throws Exception { final UnixEpochTimestampGenerator generator = new UnixEpochTimestampGenerator(120, TimeUnit.SECONDS, 1072915200L); assertEquals((long) generator.nextValue(), 1072915200L); assertEquals((long) generator.lastValue(), 1072915200L - 120); assertEquals((long) generator.nextValue(), 1072915200L + 120); assertEquals((long) generator.lastValue(), 1072915200L); } @Test public void variousIntervalsAndUnits() throws Exception { // negatives could happen, just start and roll back in time UnixEpochTimestampGenerator generator = new UnixEpochTimestampGenerator(-60, TimeUnit.SECONDS); long startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime - 60); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime - 120); assertEquals((long) generator.lastValue(), startTime - 60); generator = new UnixEpochTimestampGenerator(100, TimeUnit.NANOSECONDS); startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + 100); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + 200); assertEquals((long) generator.lastValue(), startTime + 100); generator = new UnixEpochTimestampGenerator(100, TimeUnit.MICROSECONDS); startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + 100); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + 200); assertEquals((long) generator.lastValue(), startTime + 100); generator = new UnixEpochTimestampGenerator(100, TimeUnit.MILLISECONDS); startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + 100); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + 200); assertEquals((long) generator.lastValue(), startTime + 100); generator = new UnixEpochTimestampGenerator(100, TimeUnit.SECONDS); startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + 100); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + 200); assertEquals((long) generator.lastValue(), startTime + 100); generator = new UnixEpochTimestampGenerator(1, TimeUnit.MINUTES); startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + (1 * 60)); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + (2 * 60)); assertEquals((long) generator.lastValue(), startTime + (1 * 60)); generator = new UnixEpochTimestampGenerator(1, TimeUnit.HOURS); startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + (1 * 60 * 60)); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + (2 * 60 * 60)); assertEquals((long) generator.lastValue(), startTime + (1 * 60 * 60)); generator = new UnixEpochTimestampGenerator(1, TimeUnit.DAYS); startTime = generator.currentValue(); assertEquals((long) generator.nextValue(), startTime + (1 * 60 * 60 * 24)); assertEquals((long) generator.lastValue(), startTime); assertEquals((long) generator.nextValue(), startTime + (2 * 60 * 60 * 24)); assertEquals((long) generator.lastValue(), startTime + (1 * 60 * 60 * 24)); } // TODO - With PowerMockito we could UT the initializeTimestamp(long) call. // Otherwise it would involve creating more functions and that would get ugly. } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/generator/TestZipfianGenerator.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.generator; import org.testng.annotations.Test; import static org.testng.AssertJUnit.assertFalse; public class TestZipfianGenerator { @Test public void testMinAndMaxParameter() { long min = 5; long max = 10; ZipfianGenerator zipfian = new ZipfianGenerator(min, max); for (int i = 0; i < 10000; i++) { long rnd = zipfian.nextValue(); assertFalse(rnd < min); assertFalse(rnd > max); } } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/measurements/exporter/TestMeasurementsExporter.java ================================================ /** * Copyright (c) 2015 Yahoo! Inc. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.measurements.exporter; import com.yahoo.ycsb.generator.ZipfianGenerator; import com.yahoo.ycsb.measurements.Measurements; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.testng.annotations.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Properties; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertTrue; public class TestMeasurementsExporter { @Test public void testJSONArrayMeasurementsExporter() throws IOException { Properties props = new Properties(); props.put(Measurements.MEASUREMENT_TYPE_PROPERTY, "histogram"); Measurements mm = new Measurements(props); ByteArrayOutputStream out = new ByteArrayOutputStream(); JSONArrayMeasurementsExporter export = new JSONArrayMeasurementsExporter(out); long min = 5000; long max = 100000; ZipfianGenerator zipfian = new ZipfianGenerator(min, max); for (int i = 0; i < 1000; i++) { int rnd = zipfian.nextValue().intValue(); mm.measure("UPDATE", rnd); } mm.exportMeasurements(export); export.close(); ObjectMapper mapper = new ObjectMapper(); JsonNode json = mapper.readTree(out.toString("UTF-8")); assertTrue(json.isArray()); assertEquals(json.get(0).get("measurement").asText(), "Operations"); assertEquals(json.get(4).get("measurement").asText(), "MaxLatency(us)"); assertEquals(json.get(11).get("measurement").asText(), "4"); } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/workloads/TestCoreWorkload.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.workloads; import static org.testng.Assert.assertTrue; import java.util.Properties; import org.testng.annotations.Test; import com.yahoo.ycsb.generator.DiscreteGenerator; public class TestCoreWorkload { @Test public void createOperationChooser() { final Properties p = new Properties(); p.setProperty(CoreWorkload.READ_PROPORTION_PROPERTY, "0.20"); p.setProperty(CoreWorkload.UPDATE_PROPORTION_PROPERTY, "0.20"); p.setProperty(CoreWorkload.INSERT_PROPORTION_PROPERTY, "0.20"); p.setProperty(CoreWorkload.SCAN_PROPORTION_PROPERTY, "0.20"); p.setProperty(CoreWorkload.READMODIFYWRITE_PROPORTION_PROPERTY, "0.20"); final DiscreteGenerator generator = CoreWorkload.createOperationGenerator(p); final int[] counts = new int[5]; for (int i = 0; i < 100; ++i) { switch (generator.nextString()) { case "READ": ++counts[0]; break; case "UPDATE": ++counts[1]; break; case "INSERT": ++counts[2]; break; case "SCAN": ++counts[3]; break; default: ++counts[4]; } } for (int i : counts) { // Doesn't do a wonderful job of equal distribution, but in a hundred, if we // don't see at least one of each operation then the generator is really broke. assertTrue(i > 1); } } @Test (expectedExceptions = IllegalArgumentException.class) public void createOperationChooserNullProperties() { CoreWorkload.createOperationGenerator(null); } } ================================================ FILE: core/src/test/java/com/yahoo/ycsb/workloads/TestTimeSeriesWorkload.java ================================================ /** * Copyright (c) 2017 YCSB contributors All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.workloads; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.Vector; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Client; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.NumericByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.Utils; import com.yahoo.ycsb.WorkloadException; import com.yahoo.ycsb.measurements.Measurements; import org.testng.annotations.Test; public class TestTimeSeriesWorkload { @Test public void twoThreads() throws Exception { final Properties p = getUTProperties(); Measurements.setProperties(p); final TimeSeriesWorkload wl = new TimeSeriesWorkload(); wl.init(p); Object threadState = wl.initThread(p, 0, 2); MockDB db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); long timestamp = 1451606400; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.keys.get(i), "AAAA"); assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertNotNull(db.values.get(i).get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); timestamp += 60; } } threadState = wl.initThread(p, 1, 2); db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); timestamp = 1451606400; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.keys.get(i), "AAAB"); assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertNotNull(db.values.get(i).get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); timestamp += 60; } } } @Test (expectedExceptions = WorkloadException.class) public void badTimeUnit() throws Exception { final Properties p = new Properties(); p.put(TimeSeriesWorkload.TIMESTAMP_UNITS_PROPERTY, "foobar"); getWorkload(p, true); } @Test (expectedExceptions = WorkloadException.class) public void failedToInitWorkloadBeforeThreadInit() throws Exception { final Properties p = getUTProperties(); final TimeSeriesWorkload wl = getWorkload(p, false); //wl.init(p); // <-- we NEED this :( final Object threadState = wl.initThread(p, 0, 2); final MockDB db = new MockDB(); wl.doInsert(db, threadState); } @Test (expectedExceptions = IllegalStateException.class) public void failedToInitThread() throws Exception { final Properties p = getUTProperties(); final TimeSeriesWorkload wl = getWorkload(p, true); final MockDB db = new MockDB(); wl.doInsert(db, null); } @Test public void insertOneKeyTwoTagsLowCardinality() throws Exception { final Properties p = getUTProperties(); p.put(CoreWorkload.FIELD_COUNT_PROPERTY, "1"); final TimeSeriesWorkload wl = getWorkload(p, true); final Object threadState = wl.initThread(p, 0, 1); final MockDB db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); long timestamp = 1451606400; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.keys.get(i), "AAAA"); assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertTrue(((NumericByteIterator) db.values.get(i) .get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)).isFloatingPoint()); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); timestamp += 60; } } } @Test public void insertTwoKeysTwoTagsLowCardinality() throws Exception { final Properties p = getUTProperties(); final TimeSeriesWorkload wl = getWorkload(p, true); final Object threadState = wl.initThread(p, 0, 1); final MockDB db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); long timestamp = 1451606400; int metricCtr = 0; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertTrue(((NumericByteIterator) db.values.get(i) .get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)).isFloatingPoint()); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); } if (metricCtr++ > 1) { assertEquals(db.keys.get(i), "AAAB"); if (metricCtr >= 4) { metricCtr = 0; timestamp += 60; } } else { assertEquals(db.keys.get(i), "AAAA"); } } } @Test public void insertTwoKeysTwoThreads() throws Exception { final Properties p = getUTProperties(); final TimeSeriesWorkload wl = getWorkload(p, true); Object threadState = wl.initThread(p, 0, 2); MockDB db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); long timestamp = 1451606400; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.keys.get(i), "AAAA"); // <-- key 1 assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertTrue(((NumericByteIterator) db.values.get(i) .get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)).isFloatingPoint()); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); timestamp += 60; } } threadState = wl.initThread(p, 1, 2); db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); timestamp = 1451606400; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.keys.get(i), "AAAB"); // <-- key 2 assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertNotNull(db.values.get(i).get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); timestamp += 60; } } } @Test public void insertThreeKeysTwoThreads() throws Exception { // To make sure the distribution doesn't miss any metrics final Properties p = getUTProperties(); p.put(CoreWorkload.FIELD_COUNT_PROPERTY, "3"); final TimeSeriesWorkload wl = getWorkload(p, true); Object threadState = wl.initThread(p, 0, 2); MockDB db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); long timestamp = 1451606400; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.keys.get(i), "AAAA"); assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertTrue(((NumericByteIterator) db.values.get(i) .get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)).isFloatingPoint()); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); timestamp += 60; } } threadState = wl.initThread(p, 1, 2); db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } timestamp = 1451606400; int metricCtr = 0; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertNotNull(db.values.get(i).get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); } if (metricCtr++ > 1) { assertEquals(db.keys.get(i), "AAAC"); if (metricCtr >= 4) { metricCtr = 0; timestamp += 60; } } else { assertEquals(db.keys.get(i), "AAAB"); } } } @Test public void insertWithValidation() throws Exception { final Properties p = getUTProperties(); p.put(CoreWorkload.FIELD_COUNT_PROPERTY, "1"); p.put(CoreWorkload.DATA_INTEGRITY_PROPERTY, "true"); p.put(TimeSeriesWorkload.VALUE_TYPE_PROPERTY, "integers"); final TimeSeriesWorkload wl = getWorkload(p, true); final Object threadState = wl.initThread(p, 0, 1); final MockDB db = new MockDB(); for (int i = 0; i < 74; i++) { assertTrue(wl.doInsert(db, threadState)); } assertEquals(db.keys.size(), 74); assertEquals(db.values.size(), 74); long timestamp = 1451606400; for (int i = 0; i < db.keys.size(); i++) { assertEquals(db.keys.get(i), "AAAA"); assertEquals(db.values.get(i).get("AA").toString(), "AAAA"); assertEquals(Utils.bytesToLong(db.values.get(i).get( TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT).toArray()), timestamp); assertFalse(((NumericByteIterator) db.values.get(i) .get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)).isFloatingPoint()); // validation check final TreeMap validationTags = new TreeMap(); for (final Entry entry : db.values.get(i).entrySet()) { if (entry.getKey().equals(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT) || entry.getKey().equals(TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT)) { continue; } validationTags.put(entry.getKey(), entry.getValue().toString()); } assertEquals(wl.validationFunction(db.keys.get(i), timestamp, validationTags), ((NumericByteIterator) db.values.get(i).get(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT)).getLong()); if (i % 2 == 0) { assertEquals(db.values.get(i).get("AB").toString(), "AAAA"); } else { assertEquals(db.values.get(i).get("AB").toString(), "AAAB"); timestamp += 60; } } } @Test public void read() throws Exception { final Properties p = getUTProperties(); final TimeSeriesWorkload wl = getWorkload(p, true); final Object threadState = wl.initThread(p, 0, 1); final MockDB db = new MockDB(); for (int i = 0; i < 20; i++) { wl.doTransactionRead(db, threadState); } } @Test public void verifyRow() throws Exception { final Properties p = getUTProperties(); final TimeSeriesWorkload wl = getWorkload(p, true); final TreeMap validationTags = new TreeMap(); final HashMap cells = new HashMap(); validationTags.put("AA", "AAAA"); cells.put("AA", new StringByteIterator("AAAA")); validationTags.put("AB", "AAAB"); cells.put("AB", new StringByteIterator("AAAB")); long hash = wl.validationFunction("AAAA", 1451606400L, validationTags); cells.put(TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT, new NumericByteIterator(1451606400L)); cells.put(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT, new NumericByteIterator(hash)); assertEquals(wl.verifyRow("AAAA", cells), Status.OK); // tweak the last value a bit for (final ByteIterator it : cells.values()) { it.reset(); } cells.put(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT, new NumericByteIterator(hash + 1)); assertEquals(wl.verifyRow("AAAA", cells), Status.UNEXPECTED_STATE); // no value cell, returns an unexpected state for (final ByteIterator it : cells.values()) { it.reset(); } cells.remove(TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT); assertEquals(wl.verifyRow("AAAA", cells), Status.UNEXPECTED_STATE); } @Test public void validateSettingsDataIntegrity() throws Exception { Properties p = getUTProperties(); // data validation incompatibilities p.setProperty(CoreWorkload.DATA_INTEGRITY_PROPERTY, "true"); try { getWorkload(p, true); fail("Expected WorkloadException"); } catch (WorkloadException e) { } p.setProperty(TimeSeriesWorkload.VALUE_TYPE_PROPERTY, "integers"); // now it's ok p.setProperty(TimeSeriesWorkload.GROUPBY_PROPERTY, "sum"); // now it's not try { getWorkload(p, true); fail("Expected WorkloadException"); } catch (WorkloadException e) { } p.setProperty(TimeSeriesWorkload.GROUPBY_PROPERTY, ""); p.setProperty(TimeSeriesWorkload.DOWNSAMPLING_FUNCTION_PROPERTY, "sum"); p.setProperty(TimeSeriesWorkload.DOWNSAMPLING_INTERVAL_PROPERTY, "60"); try { getWorkload(p, true); fail("Expected WorkloadException"); } catch (WorkloadException e) { } p.setProperty(TimeSeriesWorkload.DOWNSAMPLING_FUNCTION_PROPERTY, ""); p.setProperty(TimeSeriesWorkload.DOWNSAMPLING_INTERVAL_PROPERTY, ""); p.setProperty(TimeSeriesWorkload.QUERY_TIMESPAN_PROPERTY, "60"); try { getWorkload(p, true); fail("Expected WorkloadException"); } catch (WorkloadException e) { } p = getUTProperties(); p.setProperty(CoreWorkload.DATA_INTEGRITY_PROPERTY, "true"); p.setProperty(TimeSeriesWorkload.VALUE_TYPE_PROPERTY, "integers"); p.setProperty(TimeSeriesWorkload.RANDOMIZE_TIMESERIES_ORDER_PROPERTY, "true"); try { getWorkload(p, true); fail("Expected WorkloadException"); } catch (WorkloadException e) { } p.setProperty(TimeSeriesWorkload.RANDOMIZE_TIMESERIES_ORDER_PROPERTY, "false"); p.setProperty(TimeSeriesWorkload.INSERT_START_PROPERTY, ""); try { getWorkload(p, true); fail("Expected WorkloadException"); } catch (WorkloadException e) { } } /** Helper method that generates unit testing defaults for the properties map */ private Properties getUTProperties() { final Properties p = new Properties(); p.put(Client.RECORD_COUNT_PROPERTY, "10"); p.put(CoreWorkload.FIELD_COUNT_PROPERTY, "2"); p.put(CoreWorkload.FIELD_LENGTH_PROPERTY, "4"); p.put(TimeSeriesWorkload.TAG_KEY_LENGTH_PROPERTY, "2"); p.put(TimeSeriesWorkload.TAG_VALUE_LENGTH_PROPERTY, "4"); p.put(TimeSeriesWorkload.TAG_COUNT_PROPERTY, "2"); p.put(TimeSeriesWorkload.TAG_CARDINALITY_PROPERTY, "1,2"); p.put(CoreWorkload.INSERT_START_PROPERTY, "1451606400"); p.put(TimeSeriesWorkload.DELAYED_SERIES_PROPERTY, "0"); p.put(TimeSeriesWorkload.RANDOMIZE_TIMESERIES_ORDER_PROPERTY, "false"); return p; } /** Helper to setup the workload for testing. */ private TimeSeriesWorkload getWorkload(final Properties p, final boolean init) throws WorkloadException { Measurements.setProperties(p); if (!init) { return new TimeSeriesWorkload(); } else { final TimeSeriesWorkload workload = new TimeSeriesWorkload(); workload.init(p); return workload; } } static class MockDB extends DB { final List keys = new ArrayList(); final List> values = new ArrayList>(); @Override public Status read(String table, String key, Set fields, Map result) { return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // TODO Auto-generated method stub return Status.OK; } @Override public Status update(String table, String key, Map values) { // TODO Auto-generated method stub return Status.OK; } @Override public Status insert(String table, String key, Map values) { keys.add(key); this.values.add(values); return Status.OK; } @Override public Status delete(String table, String key) { // TODO Auto-generated method stub return Status.OK; } public void dumpStdout() { for (int i = 0; i < keys.size(); i++) { System.out.print("[" + i + "] Key: " + keys.get(i) + " Values: {"); int x = 0; for (final Entry entry : values.get(i).entrySet()) { if (x++ > 0) { System.out.print(", "); } System.out.print("{" + entry.getKey() + " => "); if (entry.getKey().equals("YCSBV")) { System.out.print(new String(Utils.bytesToDouble(entry.getValue().toArray()) + "}")); } else if (entry.getKey().equals("YCSBTS")) { System.out.print(new String(Utils.bytesToLong(entry.getValue().toArray()) + "}")); } else { System.out.print(new String(entry.getValue().toArray()) + "}"); } } System.out.println("}"); } } } } ================================================ FILE: couchbase/README.md ================================================ # Couchbase Driver for YCSB This driver is a binding for the YCSB facilities to operate against a Couchbase Server cluster. It uses the official Couchbase Java SDK and provides a rich set of configuration options. ## Quickstart ### 1. Start Couchbase Server You need to start a single node or a cluster to point the client at. Please see [http://couchbase.com](couchbase.com) for more details and instructions. ### 2. Set up YCSB You need to clone the repository and compile everything. ``` git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ``` ### 3. Run the Workload Before you can actually run the workload, you need to "load" the data first. ``` bin/ycsb load couchbase -s -P workloads/workloada ``` Then, you can run the workload: ``` bin/ycsb run couchbase -s -P workloads/workloada ``` Please see the general instructions in the `doc` folder if you are not sure how it all works. You can apply a property (as seen in the next section) like this: ``` bin/ycsb run couchbase -s -P workloads/workloada -p couchbase.useJson=false ``` ## Scans in the CouchbaseClient The scan operation in the CouchbaseClient requires a Couchbase View to be created manually. To do this: 1. Go to the Couchbase UI, then to Views 2. Create a new development view, specify a ddoc and view name, use these in your YCSB properties. See Configuration Options below. 3. The default map code is sufficient. 4. Save, and publish this View. ## Configuration Options Since no setup is the same and the goal of YCSB is to deliver realistic benchmarks, here are some setups that you can tune. Note that if you need more flexibility (let's say a custom transcoder), you still need to extend this driver and implement the facilities on your own. You can set the following properties (with the default settings applied): - couchbase.url=http://127.0.0.1:8091/pools => The connection URL from one server. - couchbase.bucket=default => The bucket name to use. - couchbase.password= => The password of the bucket. - couchbase.checkFutures=true => If the futures should be inspected (makes ops sync). - couchbase.persistTo=0 => Observe Persistence ("PersistTo" constraint). - couchbase.replicateTo=0 => Observe Replication ("ReplicateTo" constraint). - couchbase.ddoc => The ddoc name used for scanning - couchbase.view => The view name used for scanning - couchbase.stale => How to deal with stale values in View Query for scanning. (OK, FALSE, UPDATE_AFTER) - couchbase.json=true => Use json or java serialization as target format. ================================================ FILE: couchbase/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent couchbase-binding Couchbase Binding jar com.couchbase.client couchbase-client ${couchbase.version} com.yahoo.ycsb core ${project.version} provided com.fasterxml.jackson.core jackson-databind 2.2.2 org.slf4j slf4j-api ================================================ FILE: couchbase/src/main/java/com/yahoo/ycsb/db/CouchbaseClient.java ================================================ /** * Copyright (c) 2013 - 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.couchbase.client.protocol.views.*; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import net.spy.memcached.PersistTo; import net.spy.memcached.ReplicateTo; import net.spy.memcached.internal.OperationFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.StringWriter; import java.io.Writer; import java.net.URI; import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * A class that wraps the CouchbaseClient to allow it to be interfaced with YCSB. * This class extends {@link DB} and implements the database interface used by YCSB client. */ public class CouchbaseClient extends DB { public static final String URL_PROPERTY = "couchbase.url"; public static final String BUCKET_PROPERTY = "couchbase.bucket"; public static final String PASSWORD_PROPERTY = "couchbase.password"; public static final String CHECKF_PROPERTY = "couchbase.checkFutures"; public static final String PERSIST_PROPERTY = "couchbase.persistTo"; public static final String REPLICATE_PROPERTY = "couchbase.replicateTo"; public static final String JSON_PROPERTY = "couchbase.json"; public static final String DESIGN_DOC_PROPERTY = "couchbase.ddoc"; public static final String VIEW_PROPERTY = "couchbase.view"; public static final String STALE_PROPERTY = "couchbase.stale"; public static final String SCAN_PROPERTY = "scanproportion"; public static final String STALE_PROPERTY_DEFAULT = Stale.OK.name(); public static final String SCAN_PROPERTY_DEFAULT = "0.0"; protected static final ObjectMapper JSON_MAPPER = new ObjectMapper(); private com.couchbase.client.CouchbaseClient client; private PersistTo persistTo; private ReplicateTo replicateTo; private boolean checkFutures; private boolean useJson; private String designDoc; private String viewName; private Stale stale; private View view; private final Logger log = LoggerFactory.getLogger(getClass()); @Override public void init() throws DBException { Properties props = getProperties(); String url = props.getProperty(URL_PROPERTY, "http://127.0.0.1:8091/pools"); String bucket = props.getProperty(BUCKET_PROPERTY, "default"); String password = props.getProperty(PASSWORD_PROPERTY, ""); checkFutures = props.getProperty(CHECKF_PROPERTY, "true").equals("true"); useJson = props.getProperty(JSON_PROPERTY, "true").equals("true"); persistTo = parsePersistTo(props.getProperty(PERSIST_PROPERTY, "0")); replicateTo = parseReplicateTo(props.getProperty(REPLICATE_PROPERTY, "0")); designDoc = getProperties().getProperty(DESIGN_DOC_PROPERTY); viewName = getProperties().getProperty(VIEW_PROPERTY); stale = Stale.valueOf(getProperties().getProperty(STALE_PROPERTY, STALE_PROPERTY_DEFAULT).toUpperCase()); Double scanproportion = Double.valueOf(props.getProperty(SCAN_PROPERTY, SCAN_PROPERTY_DEFAULT)); Properties systemProperties = System.getProperties(); systemProperties.put("net.spy.log.LoggerImpl", "net.spy.memcached.compat.log.SLF4JLogger"); System.setProperties(systemProperties); try { client = new com.couchbase.client.CouchbaseClient(Arrays.asList(new URI(url)), bucket, password); } catch (Exception e) { throw new DBException("Could not create CouchbaseClient object.", e); } if (scanproportion > 0) { try { view = client.getView(designDoc, viewName); } catch (Exception e) { throw new DBException(String.format("%s=%s and %s=%s provided, unable to connect to view.", DESIGN_DOC_PROPERTY, designDoc, VIEW_PROPERTY, viewName), e.getCause()); } } } /** * Parse the replicate property into the correct enum. * * @param property the stringified property value. * @throws DBException if parsing the property did fail. * @return the correct enum. */ private ReplicateTo parseReplicateTo(final String property) throws DBException { int value = Integer.parseInt(property); switch (value) { case 0: return ReplicateTo.ZERO; case 1: return ReplicateTo.ONE; case 2: return ReplicateTo.TWO; case 3: return ReplicateTo.THREE; default: throw new DBException(REPLICATE_PROPERTY + " must be between 0 and 3"); } } /** * Parse the persist property into the correct enum. * * @param property the stringified property value. * @throws DBException if parsing the property did fail. * @return the correct enum. */ private PersistTo parsePersistTo(final String property) throws DBException { int value = Integer.parseInt(property); switch (value) { case 0: return PersistTo.ZERO; case 1: return PersistTo.ONE; case 2: return PersistTo.TWO; case 3: return PersistTo.THREE; case 4: return PersistTo.FOUR; default: throw new DBException(PERSIST_PROPERTY + " must be between 0 and 4"); } } /** * Shutdown the client. */ @Override public void cleanup() { client.shutdown(); } @Override public Status read(final String table, final String key, final Set fields, final Map result) { String formattedKey = formatKey(table, key); try { Object loaded = client.get(formattedKey); if (loaded == null) { return Status.ERROR; } decode(loaded, fields, result); return Status.OK; } catch (Exception e) { if (log.isErrorEnabled()) { log.error("Could not read value for key " + formattedKey, e); } return Status.ERROR; } } @Override public Status scan(final String table, final String startkey, final int recordcount, final Set fields, final Vector> result) { try { Query query = new Query().setRangeStart(startkey) .setLimit(recordcount) .setIncludeDocs(true) .setStale(stale); ViewResponse response = client.query(view, query); for (ViewRow row : response) { HashMap rowMap = new HashMap(); decode(row.getDocument(), fields, rowMap); result.add(rowMap); } return Status.OK; } catch (Exception e) { log.error(e.getMessage()); } return Status.ERROR; } @Override public Status update(final String table, final String key, final Map values) { String formattedKey = formatKey(table, key); try { final OperationFuture future = client.replace(formattedKey, encode(values), persistTo, replicateTo); return checkFutureStatus(future); } catch (Exception e) { if (log.isErrorEnabled()) { log.error("Could not update value for key " + formattedKey, e); } return Status.ERROR; } } @Override public Status insert(final String table, final String key, final Map values) { String formattedKey = formatKey(table, key); try { final OperationFuture future = client.add(formattedKey, encode(values), persistTo, replicateTo); return checkFutureStatus(future); } catch (Exception e) { if (log.isErrorEnabled()) { log.error("Could not insert value for key " + formattedKey, e); } return Status.ERROR; } } @Override public Status delete(final String table, final String key) { String formattedKey = formatKey(table, key); try { final OperationFuture future = client.delete(formattedKey, persistTo, replicateTo); return checkFutureStatus(future); } catch (Exception e) { if (log.isErrorEnabled()) { log.error("Could not delete value for key " + formattedKey, e); } return Status.ERROR; } } /** * Prefix the key with the given prefix, to establish a unique namespace. * * @param prefix the prefix to use. * @param key the actual key. * @return the formatted and prefixed key. */ private String formatKey(final String prefix, final String key) { return prefix + ":" + key; } /** * Wrapper method that either inspects the future or not. * * @param future the future to potentially verify. * @return the status of the future result. */ private Status checkFutureStatus(final OperationFuture future) { if (checkFutures) { return future.getStatus().isSuccess() ? Status.OK : Status.ERROR; } else { return Status.OK; } } /** * Decode the object from server into the storable result. * * @param source the loaded object. * @param fields the fields to check. * @param dest the result passed back to the ycsb core. */ private void decode(final Object source, final Set fields, final Map dest) { if (useJson) { try { JsonNode json = JSON_MAPPER.readTree((String) source); boolean checkFields = fields != null && !fields.isEmpty(); for (Iterator> jsonFields = json.fields(); jsonFields.hasNext();) { Map.Entry jsonField = jsonFields.next(); String name = jsonField.getKey(); if (checkFields && fields.contains(name)) { continue; } JsonNode jsonValue = jsonField.getValue(); if (jsonValue != null && !jsonValue.isNull()) { dest.put(name, new StringByteIterator(jsonValue.asText())); } } } catch (Exception e) { throw new RuntimeException("Could not decode JSON"); } } else { Map converted = (HashMap) source; for (Map.Entry entry : converted.entrySet()) { dest.put(entry.getKey(), new StringByteIterator(entry.getValue())); } } } /** * Encode the object for couchbase storage. * * @param source the source value. * @return the storable object. */ private Object encode(final Map source) { Map stringMap = StringByteIterator.getStringMap(source); if (!useJson) { return stringMap; } ObjectNode node = JSON_MAPPER.createObjectNode(); for (Map.Entry pair : stringMap.entrySet()) { node.put(pair.getKey(), pair.getValue()); } JsonFactory jsonFactory = new JsonFactory(); Writer writer = new StringWriter(); try { JsonGenerator jsonGenerator = jsonFactory.createGenerator(writer); JSON_MAPPER.writeTree(jsonGenerator, node); } catch (Exception e) { throw new RuntimeException("Could not encode JSON value"); } return writer.toString(); } } ================================================ FILE: couchbase/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Couchbase. */ package com.yahoo.ycsb.db; ================================================ FILE: couchbase2/README.md ================================================ # Couchbase (SDK 2.x) Driver for YCSB This driver is a binding for the YCSB facilities to operate against a Couchbase Server cluster. It uses the official Couchbase Java SDK (version 2.x) and provides a rich set of configuration options, including support for the N1QL query language. ## Quickstart ### 1. Start Couchbase Server You need to start a single node or a cluster to point the client at. Please see [http://couchbase.com](couchbase.com) for more details and instructions. ### 2. Set up YCSB You can either download the release zip and run it, or just clone from master. ``` git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ``` ### 3. Run the Workload Before you can actually run the workload, you need to "load" the data first. ``` bin/ycsb load couchbase2 -s -P workloads/workloada ``` Then, you can run the workload: ``` bin/ycsb run couchbase2 -s -P workloads/workloada ``` Please see the general instructions in the `doc` folder if you are not sure how it all works. You can apply a property (as seen in the next section) like this: ``` bin/ycsb run couchbase -s -P workloads/workloada -p couchbase.epoll=true ``` ## N1QL Index Setup In general, every time N1QL is used (either implicitly through using `workloade` or through setting `kv=false`) some kind of index must be present to make it work. Depending on the workload and data size, choosing the right index is crucial at runtime in order to get the best performance. If in doubt, please ask at the [forums](http://forums.couchbase.com) or get in touch with our team at Couchbase. For `workloade` and the default `readallfields=true` we recommend creating the following index, and if using Couchbase Server 4.5 or later with the "Memory Optimized Index" setting on the bucket. ``` CREATE PRIMARY INDEX ON `bucketname`; ``` Couchbase Server prior to 4.5 may need a slightly different index to deliver the best performance. In those releases additional covering information may be added to the index with this form. ``` -CREATE INDEX wle_idx ON `bucketname`(meta().id); ``` For other workloads, different index setups might be even more performant. ## Performance Considerations As it is with any benchmark, there are lot of knobs to tune in order to get great or (if you are reading this and trying to write a competitor benchmark ;-)) bad performance. The first setting you should consider, if you are running on Linux 64bit is setting `-p couchbase.epoll=true`. This will then turn on the Epoll IO mechanisms in the underlying Netty library which provides better performance since it has less synchronization to do than the NIO default. This only works on Linux, but you are benchmarking on the OS you are deploying to, right? The second option, `boost`, sounds more magic than it actually is. By default this benchmark trades CPU for throughput, but this can be disabled by setting `-p couchbase.boost=0`. This defaults to 3, and 3 is the number of event loops run in the IO layer. 3 is a reasonable default but you should set it to the number of **physical** cores you have available on the machine if you only plan to run one YCSB instance. Make sure (using profiling) to max out your cores, but don't overdo it. ## Sync vs Async By default, since YCSB is sync the code will always wait for the operation to complete. In some cases it can be useful to just "drive load" and disable the waiting. Note that when the "-p couchbase.syncMutationResponse=false" option is used, the measured results by YCSB can basically be thrown away. Still helpful sometimes during load phases to speed them up :) ## Debugging Latency The Couchbase Java SDK has the ability to collect and dump different kinds of metrics which allow you to analyze performance during benchmarking and production. By default this option is disabled in the benchmark, but by setting `couchbase.networkMetricsInterval` and/or `couchbase.runtimeMetricsInterval` to something greater than 0 it will output the information as JSON into the configured logger. The number provides is the interval in seconds. If you are unsure what interval to pick, start with 10 or 30 seconds, depending on your runtime length. This is how such logs look like: ``` INFO: {"heap.used":{"init":268435456,"used":36500912,"committed":232259584,"max":3817865216},"gc.ps marksweep.collectionTime":0,"gc.ps scavenge.collectionTime":54,"gc.ps scavenge.collectionCount":17,"thread.count":26,"offHeap.used":{"init":2555904,"used":30865944,"committed":31719424,"max":-1},"gc.ps marksweep.collectionCount":0,"heap.pendingFinalize":0,"thread.peakCount":26,"event":{"name":"RuntimeMetrics","type":"METRIC"},"thread.startedCount":28} INFO: {"localhost/127.0.0.1:11210":{"BINARY":{"ReplaceRequest":{"SUCCESS":{"metrics":{"percentiles":{"50.0":102,"90.0":136,"95.0":155,"99.0":244,"99.9":428},"min":55,"max":1564,"count":35787,"timeUnit":"MICROSECONDS"}}},"GetRequest":{"SUCCESS":{"metrics":{"percentiles":{"50.0":74,"90.0":98,"95.0":110,"99.0":158,"99.9":358},"min":34,"max":2310,"count":35604,"timeUnit":"MICROSECONDS"}}},"GetBucketConfigRequest":{"SUCCESS":{"metrics":{"percentiles":{"50.0":462,"90.0":462,"95.0":462,"99.0":462,"99.9":462},"min":460,"max":462,"count":1,"timeUnit":"MICROSECONDS"}}}}},"event":{"name":"NetworkLatencyMetrics","type":"METRIC"}} ``` It is recommended to either feed it into a program which can analyze and visualize JSON or just dump it into a JSON pretty printer and look at it manually. Since the output can be changed (only by changing the code at the moment), you can even configure to put those messages into another couchbase bucket and then analyze it through N1QL! You can learn more about this in general [in the official docs](http://developer.couchbase.com/documentation/server/4.0/sdks/java-2.2/event-bus-metrics.html). ## Configuration Options Since no setup is the same and the goal of YCSB is to deliver realistic benchmarks, here are some setups that you can tune. Note that if you need more flexibility (let's say a custom transcoder), you still need to extend this driver and implement the facilities on your own. You can set the following properties (with the default settings applied): - couchbase.host=127.0.0.1: The hostname from one server. - couchbase.bucket=default: The bucket name to use. - couchbase.password=: The password of the bucket. - couchbase.syncMutationResponse=true: If mutations should wait for the response to complete. - couchbase.persistTo=0: Persistence durability requirement - couchbase.replicateTo=0: Replication durability requirement - couchbase.upsert=false: Use upsert instead of insert or replace. - couchbase.adhoc=false: If set to true, prepared statements are not used. - couchbase.kv=true: If set to false, mutation operations will also be performed through N1QL. - couchbase.maxParallelism=1: The server parallelism for all n1ql queries. - couchbase.kvEndpoints=1: The number of KV sockets to open per server. - couchbase.queryEndpoints=5: The number of N1QL Query sockets to open per server. - couchbase.epoll=false: If Epoll instead of NIO should be used (only available for linux. - couchbase.boost=3: If > 0 trades CPU for higher throughput. N is the number of event loops, ideally set to the number of physical cores. Setting higher than that will likely degrade performance. - couchbase.networkMetricsInterval=0: The interval in seconds when latency metrics will be logged. - couchbase.runtimeMetricsInterval=0: The interval in seconds when runtime metrics will be logged. - couchbase.documentExpiry=0: Document Expiry is the amount of time(second) until a document expires in Couchbase. ================================================ FILE: couchbase2/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent couchbase2-binding Couchbase Java SDK 2.x Binding jar com.couchbase.client java-client ${couchbase2.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: couchbase2/src/main/java/com/yahoo/ycsb/db/couchbase2/Couchbase2Client.java ================================================ /** * Copyright (c) 2016 Yahoo! Inc. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.couchbase2; import com.couchbase.client.core.env.DefaultCoreEnvironment; import com.couchbase.client.core.env.resources.IoPoolShutdownHook; import com.couchbase.client.core.logging.CouchbaseLogger; import com.couchbase.client.core.logging.CouchbaseLoggerFactory; import com.couchbase.client.core.metrics.DefaultLatencyMetricsCollectorConfig; import com.couchbase.client.core.metrics.DefaultMetricsCollectorConfig; import com.couchbase.client.core.metrics.LatencyMetricsCollectorConfig; import com.couchbase.client.core.metrics.MetricsCollectorConfig; import com.couchbase.client.deps.com.fasterxml.jackson.core.JsonFactory; import com.couchbase.client.deps.com.fasterxml.jackson.core.JsonGenerator; import com.couchbase.client.deps.com.fasterxml.jackson.databind.JsonNode; import com.couchbase.client.deps.com.fasterxml.jackson.databind.node.ObjectNode; import com.couchbase.client.deps.io.netty.channel.DefaultSelectStrategyFactory; import com.couchbase.client.deps.io.netty.channel.EventLoopGroup; import com.couchbase.client.deps.io.netty.channel.SelectStrategy; import com.couchbase.client.deps.io.netty.channel.SelectStrategyFactory; import com.couchbase.client.deps.io.netty.channel.epoll.EpollEventLoopGroup; import com.couchbase.client.deps.io.netty.channel.nio.NioEventLoopGroup; import com.couchbase.client.deps.io.netty.util.IntSupplier; import com.couchbase.client.deps.io.netty.util.concurrent.DefaultThreadFactory; import com.couchbase.client.java.Bucket; import com.couchbase.client.java.Cluster; import com.couchbase.client.java.CouchbaseCluster; import com.couchbase.client.java.PersistTo; import com.couchbase.client.java.ReplicateTo; import com.couchbase.client.java.document.Document; import com.couchbase.client.java.document.RawJsonDocument; import com.couchbase.client.java.document.json.JsonArray; import com.couchbase.client.java.document.json.JsonObject; import com.couchbase.client.java.env.CouchbaseEnvironment; import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; import com.couchbase.client.java.error.TemporaryFailureException; import com.couchbase.client.java.query.*; import com.couchbase.client.java.transcoder.JacksonTransformers; import com.couchbase.client.java.util.Blocking; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import rx.Observable; import rx.Subscriber; import rx.functions.Action1; import rx.functions.Func1; import java.io.StringWriter; import java.io.Writer; import java.nio.channels.spi.SelectorProvider; import java.util.*; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; /** * A class that wraps the 2.x Couchbase SDK to be used with YCSB. * *

The following options can be passed when using this database client to override the defaults. * *

    *
  • couchbase.host=127.0.0.1 The hostname from one server.
    • *
  • couchbase.bucket=default The bucket name to use.
    • *
  • couchbase.password= The password of the bucket.
    • *
  • couchbase.syncMutationResponse=true If mutations should wait for the response to complete.
    • *
  • couchbase.persistTo=0 Persistence durability requirement
    • *
  • couchbase.replicateTo=0 Replication durability requirement
    • *
  • couchbase.upsert=false Use upsert instead of insert or replace.
    • *
  • couchbase.adhoc=false If set to true, prepared statements are not used.
    • *
  • couchbase.kv=true If set to false, mutation operations will also be performed through N1QL.
    • *
  • couchbase.maxParallelism=1 The server parallelism for all n1ql queries.
    • *
  • couchbase.kvEndpoints=1 The number of KV sockets to open per server.
    • *
  • couchbase.queryEndpoints=5 The number of N1QL Query sockets to open per server.
    • *
  • couchbase.epoll=false If Epoll instead of NIO should be used (only available for linux.
    • *
  • couchbase.boost=3 If > 0 trades CPU for higher throughput. N is the number of event loops, ideally * set to the number of physical cores. Setting higher than that will likely degrade performance.
    • *
  • couchbase.networkMetricsInterval=0 The interval in seconds when latency metrics will be logged.
    • *
  • couchbase.runtimeMetricsInterval=0 The interval in seconds when runtime metrics will be logged.
    • *
  • couchbase.documentExpiry=0 Document Expiry is the amount of time until a document expires in * Couchbase.
    • *
*/ public class Couchbase2Client extends DB { static { // No need to send the full encoded_plan for this benchmark workload, less network overhead! System.setProperty("com.couchbase.query.encodedPlanEnabled", "false"); } private static final String SEPARATOR = ":"; private static final CouchbaseLogger LOGGER = CouchbaseLoggerFactory.getInstance(Couchbase2Client.class); private static final Object INIT_COORDINATOR = new Object(); private static volatile CouchbaseEnvironment env = null; private Cluster cluster; private Bucket bucket; private String bucketName; private boolean upsert; private PersistTo persistTo; private ReplicateTo replicateTo; private boolean syncMutResponse; private boolean epoll; private long kvTimeout; private boolean adhoc; private boolean kv; private int maxParallelism; private String host; private int kvEndpoints; private int queryEndpoints; private int boost; private int networkMetricsInterval; private int runtimeMetricsInterval; private String scanAllQuery; private int documentExpiry; @Override public void init() throws DBException { Properties props = getProperties(); host = props.getProperty("couchbase.host", "127.0.0.1"); bucketName = props.getProperty("couchbase.bucket", "default"); String bucketPassword = props.getProperty("couchbase.password", ""); upsert = props.getProperty("couchbase.upsert", "false").equals("true"); persistTo = parsePersistTo(props.getProperty("couchbase.persistTo", "0")); replicateTo = parseReplicateTo(props.getProperty("couchbase.replicateTo", "0")); syncMutResponse = props.getProperty("couchbase.syncMutationResponse", "true").equals("true"); adhoc = props.getProperty("couchbase.adhoc", "false").equals("true"); kv = props.getProperty("couchbase.kv", "true").equals("true"); maxParallelism = Integer.parseInt(props.getProperty("couchbase.maxParallelism", "1")); kvEndpoints = Integer.parseInt(props.getProperty("couchbase.kvEndpoints", "1")); queryEndpoints = Integer.parseInt(props.getProperty("couchbase.queryEndpoints", "1")); epoll = props.getProperty("couchbase.epoll", "false").equals("true"); boost = Integer.parseInt(props.getProperty("couchbase.boost", "3")); networkMetricsInterval = Integer.parseInt(props.getProperty("couchbase.networkMetricsInterval", "0")); runtimeMetricsInterval = Integer.parseInt(props.getProperty("couchbase.runtimeMetricsInterval", "0")); documentExpiry = Integer.parseInt(props.getProperty("couchbase.documentExpiry", "0")); scanAllQuery = "SELECT RAW meta().id FROM `" + bucketName + "` WHERE meta().id >= '$1' ORDER BY meta().id LIMIT $2"; try { synchronized (INIT_COORDINATOR) { if (env == null) { LatencyMetricsCollectorConfig latencyConfig = networkMetricsInterval <= 0 ? DefaultLatencyMetricsCollectorConfig.disabled() : DefaultLatencyMetricsCollectorConfig .builder() .emitFrequency(networkMetricsInterval) .emitFrequencyUnit(TimeUnit.SECONDS) .build(); MetricsCollectorConfig runtimeConfig = runtimeMetricsInterval <= 0 ? DefaultMetricsCollectorConfig.disabled() : DefaultMetricsCollectorConfig.create(runtimeMetricsInterval, TimeUnit.SECONDS); DefaultCouchbaseEnvironment.Builder builder = DefaultCouchbaseEnvironment .builder() .queryEndpoints(queryEndpoints) .callbacksOnIoPool(true) .runtimeMetricsCollectorConfig(runtimeConfig) .networkLatencyMetricsCollectorConfig(latencyConfig) .socketConnectTimeout(10000) // 10 secs socket connect timeout .connectTimeout(30000) // 30 secs overall bucket open timeout .kvTimeout(10000) // 10 instead of 2.5s for KV ops .kvEndpoints(kvEndpoints); // Tune boosting and epoll based on settings SelectStrategyFactory factory = boost > 0 ? new BackoffSelectStrategyFactory() : DefaultSelectStrategyFactory.INSTANCE; int poolSize = boost > 0 ? boost : Integer.parseInt( System.getProperty("com.couchbase.ioPoolSize", Integer.toString(DefaultCoreEnvironment.IO_POOL_SIZE)) ); ThreadFactory threadFactory = new DefaultThreadFactory("cb-io", true); EventLoopGroup group = epoll ? new EpollEventLoopGroup(poolSize, threadFactory, factory) : new NioEventLoopGroup(poolSize, threadFactory, SelectorProvider.provider(), factory); builder.ioPool(group, new IoPoolShutdownHook(group)); env = builder.build(); logParams(); } } cluster = CouchbaseCluster.create(env, host); bucket = cluster.openBucket(bucketName, bucketPassword); kvTimeout = env.kvTimeout(); } catch (Exception ex) { throw new DBException("Could not connect to Couchbase Bucket.", ex); } if (!kv && !syncMutResponse) { throw new DBException("Not waiting for N1QL responses on mutations not yet implemented."); } } /** * Helper method to log the CLI params so that on the command line debugging is easier. */ private void logParams() { StringBuilder sb = new StringBuilder(); sb.append("host=").append(host); sb.append(", bucket=").append(bucketName); sb.append(", upsert=").append(upsert); sb.append(", persistTo=").append(persistTo); sb.append(", replicateTo=").append(replicateTo); sb.append(", syncMutResponse=").append(syncMutResponse); sb.append(", adhoc=").append(adhoc); sb.append(", kv=").append(kv); sb.append(", maxParallelism=").append(maxParallelism); sb.append(", queryEndpoints=").append(queryEndpoints); sb.append(", kvEndpoints=").append(kvEndpoints); sb.append(", queryEndpoints=").append(queryEndpoints); sb.append(", epoll=").append(epoll); sb.append(", boost=").append(boost); sb.append(", networkMetricsInterval=").append(networkMetricsInterval); sb.append(", runtimeMetricsInterval=").append(runtimeMetricsInterval); LOGGER.info("===> Using Params: " + sb.toString()); } @Override public Status read(final String table, final String key, Set fields, final Map result) { try { String docId = formatId(table, key); if (kv) { return readKv(docId, fields, result); } else { return readN1ql(docId, fields, result); } } catch (Exception ex) { ex.printStackTrace(); return Status.ERROR; } } /** * Performs the {@link #read(String, String, Set, Map)} operation via Key/Value ("get"). * * @param docId the document ID * @param fields the fields to be loaded * @param result the result map where the doc needs to be converted into * @return The result of the operation. */ private Status readKv(final String docId, final Set fields, final Map result) throws Exception { RawJsonDocument loaded = bucket.get(docId, RawJsonDocument.class); if (loaded == null) { return Status.NOT_FOUND; } decode(loaded.content(), fields, result); return Status.OK; } /** * Performs the {@link #read(String, String, Set, Map)} operation via N1QL ("SELECT"). * * If this option should be used, the "-p couchbase.kv=false" property must be set. * * @param docId the document ID * @param fields the fields to be loaded * @param result the result map where the doc needs to be converted into * @return The result of the operation. */ private Status readN1ql(final String docId, Set fields, final Map result) throws Exception { String readQuery = "SELECT " + joinFields(fields) + " FROM `" + bucketName + "` USE KEYS [$1]"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( readQuery, JsonArray.from(docId), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new DBException("Error while parsing N1QL Result. Query: " + readQuery + ", Errors: " + queryResult.errors()); } N1qlQueryRow row; try { row = queryResult.rows().next(); } catch (NoSuchElementException ex) { return Status.NOT_FOUND; } JsonObject content = row.value(); if (fields == null) { content = content.getObject(bucketName); // n1ql result set scoped under *.bucketName fields = content.getNames(); } for (String field : fields) { Object value = content.get(field); result.put(field, new StringByteIterator(value != null ? value.toString() : "")); } return Status.OK; } @Override public Status update(final String table, final String key, final Map values) { if (upsert) { return upsert(table, key, values); } try { String docId = formatId(table, key); if (kv) { return updateKv(docId, values); } else { return updateN1ql(docId, values); } } catch (Exception ex) { ex.printStackTrace(); return Status.ERROR; } } /** * Performs the {@link #update(String, String, Map)} operation via Key/Value ("replace"). * * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status updateKv(final String docId, final Map values) { waitForMutationResponse(bucket.async().replace( RawJsonDocument.create(docId, documentExpiry, encode(values)), persistTo, replicateTo )); return Status.OK; } /** * Performs the {@link #update(String, String, Map)} operation via N1QL ("UPDATE"). * * If this option should be used, the "-p couchbase.kv=false" property must be set. * * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status updateN1ql(final String docId, final Map values) throws Exception { String fields = encodeN1qlFields(values); String updateQuery = "UPDATE `" + bucketName + "` USE KEYS [$1] SET " + fields; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( updateQuery, JsonArray.from(docId), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new DBException("Error while parsing N1QL Result. Query: " + updateQuery + ", Errors: " + queryResult.errors()); } return Status.OK; } @Override public Status insert(final String table, final String key, final Map values) { if (upsert) { return upsert(table, key, values); } try { String docId = formatId(table, key); if (kv) { return insertKv(docId, values); } else { return insertN1ql(docId, values); } } catch (Exception ex) { ex.printStackTrace(); return Status.ERROR; } } /** * Performs the {@link #insert(String, String, Map)} operation via Key/Value ("INSERT"). * * Note that during the "load" phase it makes sense to retry TMPFAILS (so that even if the server is * overloaded temporarily the ops will succeed eventually). The current code will retry TMPFAILs * for maximum of one minute and then bubble up the error. * * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status insertKv(final String docId, final Map values) { int tries = 60; // roughly 60 seconds with the 1 second sleep, not 100% accurate. for(int i = 0; i < tries; i++) { try { waitForMutationResponse(bucket.async().insert( RawJsonDocument.create(docId, documentExpiry, encode(values)), persistTo, replicateTo )); return Status.OK; } catch (TemporaryFailureException ex) { try { Thread.sleep(1000); } catch (InterruptedException e) { throw new RuntimeException("Interrupted while sleeping on TMPFAIL backoff.", ex); } } } throw new RuntimeException("Still receiving TMPFAIL from the server after trying " + tries + " times. " + "Check your server."); } /** * Performs the {@link #insert(String, String, Map)} operation via N1QL ("INSERT"). * * If this option should be used, the "-p couchbase.kv=false" property must be set. * * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status insertN1ql(final String docId, final Map values) throws Exception { String insertQuery = "INSERT INTO `" + bucketName + "`(KEY,VALUE) VALUES ($1,$2)"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( insertQuery, JsonArray.from(docId, valuesToJsonObject(values)), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new DBException("Error while parsing N1QL Result. Query: " + insertQuery + ", Errors: " + queryResult.errors()); } return Status.OK; } /** * Performs an upsert instead of insert or update using either Key/Value or N1QL. * * If this option should be used, the "-p couchbase.upsert=true" property must be set. * * @param table The name of the table * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return The result of the operation. */ private Status upsert(final String table, final String key, final Map values) { try { String docId = formatId(table, key); if (kv) { return upsertKv(docId, values); } else { return upsertN1ql(docId, values); } } catch (Exception ex) { ex.printStackTrace(); return Status.ERROR; } } /** * Performs the {@link #upsert(String, String, Map)} operation via Key/Value ("upsert"). * * If this option should be used, the "-p couchbase.upsert=true" property must be set. * * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status upsertKv(final String docId, final Map values) { waitForMutationResponse(bucket.async().upsert( RawJsonDocument.create(docId, documentExpiry, encode(values)), persistTo, replicateTo )); return Status.OK; } /** * Performs the {@link #upsert(String, String, Map)} operation via N1QL ("UPSERT"). * * If this option should be used, the "-p couchbase.upsert=true -p couchbase.kv=false" properties must be set. * * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status upsertN1ql(final String docId, final Map values) throws Exception { String upsertQuery = "UPSERT INTO `" + bucketName + "`(KEY,VALUE) VALUES ($1,$2)"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( upsertQuery, JsonArray.from(docId, valuesToJsonObject(values)), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new DBException("Error while parsing N1QL Result. Query: " + upsertQuery + ", Errors: " + queryResult.errors()); } return Status.OK; } @Override public Status delete(final String table, final String key) { try { String docId = formatId(table, key); if (kv) { return deleteKv(docId); } else { return deleteN1ql(docId); } } catch (Exception ex) { ex.printStackTrace(); return Status.ERROR; } } /** * Performs the {@link #delete(String, String)} (String, String)} operation via Key/Value ("remove"). * * @param docId the document ID. * @return The result of the operation. */ private Status deleteKv(final String docId) { waitForMutationResponse(bucket.async().remove( docId, persistTo, replicateTo )); return Status.OK; } /** * Performs the {@link #delete(String, String)} (String, String)} operation via N1QL ("DELETE"). * * If this option should be used, the "-p couchbase.kv=false" property must be set. * * @param docId the document ID. * @return The result of the operation. */ private Status deleteN1ql(final String docId) throws Exception { String deleteQuery = "DELETE FROM `" + bucketName + "` USE KEYS [$1]"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( deleteQuery, JsonArray.from(docId), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new DBException("Error while parsing N1QL Result. Query: " + deleteQuery + ", Errors: " + queryResult.errors()); } return Status.OK; } @Override public Status scan(final String table, final String startkey, final int recordcount, final Set fields, final Vector> result) { try { if (fields == null || fields.isEmpty()) { return scanAllFields(table, startkey, recordcount, result); } else { return scanSpecificFields(table, startkey, recordcount, fields, result); } } catch (Exception ex) { ex.printStackTrace(); return Status.ERROR; } } /** * Performs the {@link #scan(String, String, int, Set, Vector)} operation, optimized for all fields. * * Since the full document bodies need to be loaded anyways, it makes sense to just grab the document IDs * from N1QL and then perform the bulk loading via KV for better performance. This is a usual pattern with * Couchbase and shows the benefits of using both N1QL and KV together. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return The result of the operation. */ private Status scanAllFields(final String table, final String startkey, final int recordcount, final Vector> result) { final List> data = new ArrayList>(recordcount); bucket.async() .query(N1qlQuery.parameterized( scanAllQuery, JsonArray.from(formatId(table, startkey), recordcount), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )) .doOnNext(new Action1() { @Override public void call(AsyncN1qlQueryResult result) { if (!result.parseSuccess()) { throw new RuntimeException("Error while parsing N1QL Result. Query: " + scanAllQuery + ", Errors: " + result.errors()); } } }) .flatMap(new Func1>() { @Override public Observable call(AsyncN1qlQueryResult result) { return result.rows(); } }) .flatMap(new Func1>() { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); } }) .map(new Func1>() { @Override public HashMap call(RawJsonDocument document) { HashMap tuple = new HashMap(); decode(document.content(), null, tuple); return tuple; } }) .toBlocking() .forEach(new Action1>() { @Override public void call(HashMap tuple) { data.add(tuple); } }); result.addAll(data); return Status.OK; } /** * Performs the {@link #scan(String, String, int, Set, Vector)} operation N1Ql only for a subset of the fields. * * @param table The name of the table * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return The result of the operation. */ private Status scanSpecificFields(final String table, final String startkey, final int recordcount, final Set fields, final Vector> result) { String scanSpecQuery = "SELECT " + joinFields(fields) + " FROM `" + bucketName + "` WHERE meta().id >= '$1' LIMIT $2"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( scanSpecQuery, JsonArray.from(formatId(table, startkey), recordcount), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new RuntimeException("Error while parsing N1QL Result. Query: " + scanSpecQuery + ", Errors: " + queryResult.errors()); } boolean allFields = fields == null || fields.isEmpty(); result.ensureCapacity(recordcount); for (N1qlQueryRow row : queryResult) { JsonObject value = row.value(); if (fields == null) { value = value.getObject(bucketName); } Set f = allFields ? value.getNames() : fields; HashMap tuple = new HashMap(f.size()); for (String field : f) { tuple.put(field, new StringByteIterator(value.getString(field))); } result.add(tuple); } return Status.OK; } /** * Helper method to block on the response, depending on the property set. * * By default, since YCSB is sync the code will always wait for the operation to complete. In some * cases it can be useful to just "drive load" and disable the waiting. Note that when the * "-p couchbase.syncMutationResponse=false" option is used, the measured results by YCSB can basically * be thrown away. Still helpful sometimes during load phases to speed them up :) * * @param input the async input observable. */ private void waitForMutationResponse(final Observable> input) { if (!syncMutResponse) { ((Observable>)input).subscribe(new Subscriber>() { @Override public void onCompleted() { } @Override public void onError(Throwable e) { } @Override public void onNext(Document document) { } }); } else { Blocking.blockForSingle(input, kvTimeout, TimeUnit.MILLISECONDS); } } /** * Helper method to turn the values into a String, used with {@link #upsertN1ql(String, Map)}. * * @param values the values to encode. * @return the encoded string. */ private static String encodeN1qlFields(final Map values) { if (values.isEmpty()) { return ""; } StringBuilder sb = new StringBuilder(); for (Map.Entry entry : values.entrySet()) { String raw = entry.getValue().toString(); String escaped = raw.replace("\"", "\\\"").replace("\'", "\\\'"); sb.append(entry.getKey()).append("=\"").append(escaped).append("\" "); } String toReturn = sb.toString(); return toReturn.substring(0, toReturn.length() - 1); } /** * Helper method to turn the map of values into a {@link JsonObject} for further use. * * @param values the values to transform. * @return the created json object. */ private static JsonObject valuesToJsonObject(final Map values) { JsonObject result = JsonObject.create(); for (Map.Entry entry : values.entrySet()) { result.put(entry.getKey(), entry.getValue().toString()); } return result; } /** * Helper method to join the set of fields into a String suitable for N1QL. * * @param fields the fields to join. * @return the joined fields as a String. */ private static String joinFields(final Set fields) { if (fields == null || fields.isEmpty()) { return "*"; } StringBuilder builder = new StringBuilder(); for (String f : fields) { builder.append("`").append(f).append("`").append(","); } String toReturn = builder.toString(); return toReturn.substring(0, toReturn.length() - 1); } /** * Helper method to turn the prefix and key into a proper document ID. * * @param prefix the prefix (table). * @param key the key itself. * @return a document ID that can be used with Couchbase. */ private static String formatId(final String prefix, final String key) { return prefix + SEPARATOR + key; } /** * Helper method to parse the "ReplicateTo" property on startup. * * @param property the proeprty to parse. * @return the parsed setting. */ private static ReplicateTo parseReplicateTo(final String property) throws DBException { int value = Integer.parseInt(property); switch (value) { case 0: return ReplicateTo.NONE; case 1: return ReplicateTo.ONE; case 2: return ReplicateTo.TWO; case 3: return ReplicateTo.THREE; default: throw new DBException("\"couchbase.replicateTo\" must be between 0 and 3"); } } /** * Helper method to parse the "PersistTo" property on startup. * * @param property the proeprty to parse. * @return the parsed setting. */ private static PersistTo parsePersistTo(final String property) throws DBException { int value = Integer.parseInt(property); switch (value) { case 0: return PersistTo.NONE; case 1: return PersistTo.ONE; case 2: return PersistTo.TWO; case 3: return PersistTo.THREE; case 4: return PersistTo.FOUR; default: throw new DBException("\"couchbase.persistTo\" must be between 0 and 4"); } } /** * Decode the String from server and pass it into the decoded destination. * * @param source the loaded object. * @param fields the fields to check. * @param dest the result passed back to YCSB. */ private void decode(final String source, final Set fields, final Map dest) { try { JsonNode json = JacksonTransformers.MAPPER.readTree(source); boolean checkFields = fields != null && !fields.isEmpty(); for (Iterator> jsonFields = json.fields(); jsonFields.hasNext();) { Map.Entry jsonField = jsonFields.next(); String name = jsonField.getKey(); if (checkFields && !fields.contains(name)) { continue; } JsonNode jsonValue = jsonField.getValue(); if (jsonValue != null && !jsonValue.isNull()) { dest.put(name, new StringByteIterator(jsonValue.asText())); } } } catch (Exception e) { throw new RuntimeException("Could not decode JSON"); } } /** * Encode the source into a String for storage. * * @param source the source value. * @return the encoded string. */ private String encode(final Map source) { Map stringMap = StringByteIterator.getStringMap(source); ObjectNode node = JacksonTransformers.MAPPER.createObjectNode(); for (Map.Entry pair : stringMap.entrySet()) { node.put(pair.getKey(), pair.getValue()); } JsonFactory jsonFactory = new JsonFactory(); Writer writer = new StringWriter(); try { JsonGenerator jsonGenerator = jsonFactory.createGenerator(writer); JacksonTransformers.MAPPER.writeTree(jsonGenerator, node); } catch (Exception e) { throw new RuntimeException("Could not encode JSON value"); } return writer.toString(); } } /** * Factory for the {@link BackoffSelectStrategy} to be used with boosting. */ class BackoffSelectStrategyFactory implements SelectStrategyFactory { @Override public SelectStrategy newSelectStrategy() { return new BackoffSelectStrategy(); } } /** * Custom IO select strategy which trades CPU for throughput, used with the boost setting. */ class BackoffSelectStrategy implements SelectStrategy { private int counter = 0; @Override public int calculateStrategy(final IntSupplier supplier, final boolean hasTasks) throws Exception { int selectNowResult = supplier.get(); if (hasTasks || selectNowResult != 0) { counter = 0; return selectNowResult; } counter++; if (counter > 2000) { LockSupport.parkNanos(1); } else if (counter > 3000) { Thread.yield(); } else if (counter > 4000) { LockSupport.parkNanos(1000); } else if (counter > 5000) { // defer to blocking select counter = 0; return SelectStrategy.SELECT; } return SelectStrategy.CONTINUE; } } ================================================ FILE: couchbase2/src/main/java/com/yahoo/ycsb/db/couchbase2/package-info.java ================================================ /* * Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Couchbase, new driver. */ package com.yahoo.ycsb.db.couchbase2; ================================================ FILE: distribution/pom.xml ================================================ 4.0.0 com.yahoo.ycsb root 0.14.0-SNAPSHOT ycsb YCSB Release Distribution Builder pom This module creates the release package of the YCSB with all DB library bindings. It is only used by the build process and does not contain any real code of itself. com.yahoo.ycsb core ${project.version} com.yahoo.ycsb accumulo1.6-binding ${project.version} com.yahoo.ycsb accumulo1.7-binding ${project.version} com.yahoo.ycsb accumulo1.8-binding ${project.version} com.yahoo.ycsb aerospike-binding ${project.version} com.yahoo.ycsb arangodb-binding ${project.version} com.yahoo.ycsb arangodb3-binding ${project.version} com.yahoo.ycsb asynchbase-binding ${project.version} com.yahoo.ycsb cassandra-binding ${project.version} com.yahoo.ycsb cloudspanner-binding ${project.version} com.yahoo.ycsb couchbase-binding ${project.version} com.yahoo.ycsb couchbase2-binding ${project.version} com.yahoo.ycsb azuredocumentdb-binding ${project.version} com.yahoo.ycsb azuretablestorage-binding ${project.version} com.yahoo.ycsb dynamodb-binding ${project.version} com.yahoo.ycsb elasticsearch-binding ${project.version} com.yahoo.ycsb elasticsearch5-binding ${project.version} com.yahoo.ycsb geode-binding ${project.version} com.yahoo.ycsb googledatastore-binding ${project.version} com.yahoo.ycsb googlebigtable-binding ${project.version} com.yahoo.ycsb hbase098-binding ${project.version} com.yahoo.ycsb hbase10-binding ${project.version} com.yahoo.ycsb hbase12-binding ${project.version} com.yahoo.ycsb hypertable-binding ${project.version} com.yahoo.ycsb infinispan-binding ${project.version} com.yahoo.ycsb jdbc-binding ${project.version} com.yahoo.ycsb kudu-binding ${project.version} com.yahoo.ycsb memcached-binding ${project.version} com.yahoo.ycsb mongodb-binding ${project.version} com.yahoo.ycsb nosqldb-binding ${project.version} com.yahoo.ycsb orientdb-binding ${project.version} com.yahoo.ycsb rados-binding ${project.version} com.yahoo.ycsb redis-binding ${project.version} com.yahoo.ycsb rest-binding ${project.version} com.yahoo.ycsb riak-binding ${project.version} com.yahoo.ycsb s3-binding ${project.version} com.yahoo.ycsb solr-binding ${project.version} com.yahoo.ycsb solr6-binding ${project.version} com.yahoo.ycsb tarantool-binding ${project.version} org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} src/main/assembly/distribution.xml false posix package single ================================================ FILE: distribution/src/main/assembly/distribution.xml ================================================ package tar.gz true .. . 0644 README LICENSE.txt NOTICE.txt ../bin bin 0755 ycsb* ../bin bin 0644 bindings.properties ../workloads workloads 0644 lib com.yahoo.ycsb:core runtime false false true true true true com.yahoo.ycsb:core com.yahoo.ycsb:binding-parent com.yahoo.ycsb:datastore-specific-descriptor com.yahoo.ycsb:ycsb com.yahoo.ycsb:root README.md conf src/main/conf lib target/dependency false ${module.artifactId}/lib false ================================================ FILE: doc/coreproperties.html ================================================ YCSB - Core workload package properties

Yahoo! Cloud Serving Benchmark

Version 0.1.2

Home - Core workloads - Tips and FAQ

Core workload package properties

The property files used with the core workload generator can specify values for the following properties:

  • fieldcount: the number of fields in a record (default: 10)
  • fieldlength: the size of each field (default: 100)
  • readallfields: should reads read all fields (true) or just one (false) (default: true)
  • readproportion: what proportion of operations should be reads (default: 0.95)
  • updateproportion: what proportion of operations should be updates (default: 0.05)
  • insertproportion: what proportion of operations should be inserts (default: 0)
  • scanproportion: what proportion of operations should be scans (default: 0)
  • readmodifywriteproportion: what proportion of operations should be read a record, modify it, write it back (default: 0)
  • requestdistribution: what distribution should be used to select the records to operate on - uniform, zipfian or latest (default: uniform)
  • maxscanlength: for scans, what is the maximum number of records to scan (default: 1000)
  • scanlengthdistribution: for scans, what distribution should be used to choose the number of records to scan, for each scan, between 1 and maxscanlength (default: uniform)
  • insertorder: should records be inserted in order by key ("ordered"), or in hashed order ("hashed") (default: hashed)
YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. ================================================ FILE: doc/coreworkloads.html ================================================ YCSB - Core workloads

Yahoo! Cloud Serving Benchmark

Version 0.1.2

Home - Core workloads - Tips and FAQ

Core workloads

YCSB includes a set of core workloads that define a basic benchmark for cloud systems. Of course, you can define your own workloads, as described here. However, the core workloads are a useful first step, and obtaining these benchmark numbers for a variety of different systems would allow you to understand the performance tradeoffs of different systems.

The core workloads consist of six different workloads:

Workload A: Update heavy workload

This workload has a mix of 50/50 reads and writes. An application example is a session store recording recent actions.

Workload B: Read mostly workload

This workload has a 95/5 reads/write mix. Application example: photo tagging; add a tag is an update, but most operations are to read tags.

Workload C: Read only

This workload is 100% read. Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop).

Workload D: Read latest workload

In this workload, new records are inserted, and the most recently inserted records are the most popular. Application example: user status updates; people want to read the latest.

Workload E: Short ranges

In this workload, short ranges of records are queried, instead of individual records. Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id).

Workload F: Read-modify-write

In this workload, the client will read a record, modify it, and write back the changes. Application example: user database, where user records are read and modified by the user or to record user activity.

Running the workloads

All six workloads have a data set which is similar. Workloads D and E insert records during the test run. Thus, to keep the database size consistent, we recommend the following sequence:
  1. Load the database, using workload A's parameter file (workloads/workloada) and the "-load" switch to the client.
  2. Run workload A (using workloads/workloada and "-t") for a variety of throughputs.
  3. Run workload B (using workloads/workloadb and "-t") for a variety of throughputs.
  4. Run workload C (using workloads/workloadc and "-t") for a variety of throughputs.
  5. Run workload F (using workloads/workloadf and "-t") for a variety of throughputs.
  6. Run workload D (using workloads/workloadd and "-t") for a variety of throughputs. This workload inserts records, increasing the size of the database.
  7. Delete the data in the database.
  8. Reload the database, using workload E's parameter file (workloads/workloade) and the "-load switch to the client.
  9. Run workload E (using workloads/workloadd and "-t") for a variety of throughputs. This workload inserts records, increasing the size of the database.
YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. ================================================ FILE: doc/dblayer.html ================================================ YCSB - DB Interface Layer

Yahoo! Cloud Serving Benchmark

Version 0.1.2

Home - Core workloads - Tips and FAQ

Implementing a database interface layer - overview

The database interface layer hides the details of the specific database you are benchmarking from the YCSB Client. This allows the client to generate operations like "read record" or "update record" without having to understand the specific API of your database. Thus, it is very easy to benchmark new database systems; once you have created the database interface layer, the rest of the benchmark framework runs without having to change.

The database interface layer is a simple abstract class that provides read, insert, update, delete and scan operations for your database. Implementing a database interface layer for your database means filling out the body of each of those methods. Once you have compiled your layer, you can specify the name of your implemented class on the command line (or as a property) to the YCSB Client. The YCSB Client will load your implementation dynamically when it starts. Thus, you do not need to recompile the YCSB Client itself to add or change a database interface layer.

Creating a new layer step-by-step

Step 1 - Extend com.yahoo.ycsb.DB

The base class of all database interface layer implementations is com.yahoo.ycsb.DB. This is an abstract class, so you need to create a new class which extends the DB class. Your class must have a public no-argument constructor, because the instances will be constructed inside a factory which will use the no-argument constructor.

The YCSB Client framework will create one instance of your DB class per worker thread, but there might be multiple worker threads generating the workload, so there might be multiple instances of your DB class created.

Step 2 - Implement init() if necessary

You can perform any initialization of your DB object by implementing the following method 
 
public void init() throws DBException
to perform any initialization actions. The init() method will be called once per DB instance; so if there are multiple threads, each DB instance will have init() called separately.

The init() method should be used to set up the connection to the database and do any other initialization. In particular, you can configure your database layer using properties passed to the YCSB Client at runtime. In fact, the YCSB Client will pass to the DB interface layer all of the properties specified in all parameter files specified when the Client starts up. Thus, you can create new properties for configuring your DB interface layer, set them in your parameter files (or on the command line), and then retrieve them inside your implementation of the DB interface layer.

These properties will be passed to the DB instance after the constructor, so it is important to retrieve them only in the init() method and not the constructor. You can get the set of properties using the 

public Properties getProperties()
method which is already implemented and inherited from the DB base class.

Step 3 - Implement the database query and update methods

The methods that you need to implement are: 
  //Read a single record
  public int read(String table, String key, Set fields, HashMap result);

  //Perform a range scan
  public int scan(String table, String startkey, int recordcount, Set fields, Vector> result);
	
  //Update a single record
  public int update(String table, String key, HashMap values);

  //Insert a single record
  public int insert(String table, String key, HashMap values);

  //Delete a single record
  public int delete(String table, String key);
In each case, the method takes a table name and record key. (In the case of scan, the record key is the first key in the range to scan.) For the read methods (read() and scan()) the methods additionally take a set of fields to be read, and provide a structure (HashMap or Vector of HashMaps) to store the returned data. For the write methods (insert() and update()) the methods take HashMap which maps field names to values.

The database should have the appropriate tables created before you run the benchmark. So you can assume in your implementation of the above methods that the appropriate tables already exist, and just write code to read or write from the tables named in the "table" parameter.

Step 4 - Compile your database interface layer

Your code can be compiled separately from the compilation of the YCSB Client and framework. In particular, you can make changes to your DB class and recompile without having to recompile the YCSB Client.

Step 5 - Use it with the YCSB Client

Make sure that the classes for your implementation (or a jar containing those classes) are available on your CLASSPATH, as well as any libraries/jar files used by your implementation. Now, when you run the YCSB Client, specify the "-db" argument on the command line and provide the fully qualified classname of your DB class. For example, to run workloada with your DB class: 
%  java -cp build/ycsb.jar:yourjarpath com.yahoo.ycsb.Client -t -db com.foo.YourDBClass -P workloads/workloada -P large.dat -s > transactions.dat
You can also specify the DB interface layer using the DB property in your parameter file: 
db=com.foo.YourDBClass
YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. ================================================ FILE: doc/index.html ================================================ YCSB - Yahoo! Cloud Serving Benchmark

Yahoo! Cloud Serving Benchmark

Version 0.1.2

Home - Core workloads - Tips and FAQ

Overview

There are many new serving databases available, including: It is difficult to decide which system is right for your application, partially because the features differ between systems, and partially because there is not an easy way to compare the performance of one system versus another.

The goal of the YCSB project is to develop a framework and common set of workloads for evaluating the performance of different "key-value" and "cloud" serving stores. The project comprises two things:

  • The YCSB Client, an extensible workload generator
  • The Core workloads, a set of workload scenarios to be executed by the generator
Although the core workloads provide a well rounded picture of a system's performance, the Client is extensible so that you can define new and different workloads to examine system aspects, or application scenarios, not adequately covered by the core workload. Similarly, the Client is extensible to support benchmarking different databases. Although we include sample code for benchmarking HBase and Cassandra, it is straightforward to write a new interface layer to benchmark your favorite database.

A common use of the tool is to benchmark multiple systems and compare them. For example, you can install multiple systems on the same hardward configuration, and run the same workloads against each system. Then you can plot the performance of each system (for example, as latency versus throughput curves) to see when one system does better than another.

Download YCSB

YCSB is available at http://wiki.github.com/brianfrankcooper/YCSB/.

Getting started

Detailed instructions for using YCSB are available on the GitHub wiki: http://wiki.github.com/brianfrankcooper/YCSB/getting-started.

Extending YCSB

YCSB is designed to be extensible. It is easy to add a new database interface layer to support benchmarking a new database. It is also easy to define new workloads. More details about the entire class structure of YCSB is available here:
YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. ================================================ FILE: doc/parallelclients.html ================================================ YCSB - Parallel clients

Yahoo! Cloud Serving Benchmark

Version 0.1.2

Home - Core workloads - Tips and FAQ

Running multiple clients in parallel

It is straightforward to run the transaction phase of the workload from multiple servers - just start up clients on different servers, each running the same workload. Each client will produce performance statistics when it is done, and you'll have to aggregate these individual files into a single set of results.

In some cases it makes sense to load the database using multiple servers. In this case, you will want to partition the records to be loaded among the clients. Normally, YCSB just loads all of the records (as defined by the recordcount property). However, if you want to partition the load you need to additionally specify two other properties for each client:

  • insertstart: The index of the record to start at.
  • insertcount: The number of records to insert.
These properties can be specified in a property file or on the command line using the -p option.

For example, imagine you want to load 100 million records (so recordcount=100000000). Imagine you want to load with four clients. For the first client: 

insertstart=0
insertcount=25000000
For the second client: 
insertstart=25000000
insertcount=25000000
For the third client: 
insertstart=50000000
insertcount=25000000
And for the fourth client: 
insertstart=75000000
insertcount=25000000
YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. ================================================ FILE: doc/tipsfaq.html ================================================ YCSB - Tips and FAQ

Yahoo! Cloud Serving Benchmark

Version 0.1.2

Home - Core workloads - Tips and FAQ

Tips

Tip 1 - Carefully adjust the number of threads

The number of threads determines how much workload you can generate against the database. Imagine that you are trying to run a test with 10,000 operations per second, but you are only achieving 8,000 operations per second. Is this because the database can't keep up with the load? Not necessarily. Imagine that you are running with 100 client threads (e.g. "-threads 100") and each operation is taking 12 milliseconds on average. Each thread will only be able to generate 83 operations per second, because each thread operates sequentially. Over 100 threads, your client will only generate 8300 operations per second, even if the database can support more. Increasing the number of threads ensures there are enough parallel clients hitting the database so that the database, not the client, is the bottleneck.

To calculate the number of threads needed, you should have some idea of the expected latency. For example, at 10,000 operations per second, we might expect the database to have a latency of 10-30 milliseconds on average. So you to generate 10,000 operations per second, you will need (Ops per sec / (1000 / avg latency in ms) ), or (10000/(1000/30))=300 threads. In fact, to be conservative, you might consider having 400 threads. Although this is a lot of threads, each thread will spend most of its time waiting for the database to respond, so the context switching overhead will be low.

Experiment with increasing the number of threads, especially if you find you are not reaching your target throughput. Eventually, of course, you will saturate the database and there will be no way to increase the number of threads to get more throughput (in fact, increasing the number of client threads may make things worse) but you need to have enough threads to ensure it is the database, not the client, that is the bottleneck.

YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. ================================================ FILE: doc/workload.html ================================================ YCSB - Implementing new workloads

Yahoo! Cloud Serving Benchmark

Version 0.1.2

Home - Core workloads - Tips and FAQ

Implementing new workloads - overview

A workload represents the load that a given application will put on the database system. For benchmarking purposes, we must define workloads that are relatively simple compared to real applications, so that we can better reason about the benchmarking results we get. However, a workload should be detailed enough so that once we measure the database's performance, we know what kinds of applications might experience similar performance.

In the context of YCSB, a workload defines both a data set, which is a set of records to be loaded into the database, and a transaction set, which are the set of read and write operations against the database. Creating the transactions requires understanding the structure of the records, which is why both the data and the transactions must be defined in the workload.

For a complete benchmark, multiple important (but distinct) workloads might be grouped together into a workload package. The CoreWorkload package included with the YCSB client is an example of such a collection of workloads.

Typically a workload consists of two files:

  • A java class which contains the code to create data records and generate transactions against them
  • A parameter file which tunes the specifics of the workload
For example, a workload class file might generate some combination of read and update operations against the database. The parameter file might specify whether the mix of reads and updates is 50/50, 80/20, etc.

There are two ways to create a new workload or package of workloads.

Option 1: new parameter files

The core workloads included with YCSB are defined by a set of parameter files (workloada, workloadb, etc.) You can create your own parameter file with new values for the read/write mix, request distribution, etc. For example, the workloada file has the following contents: 

workload=com.yahoo.ycsb.workloads.CoreWorkload

readallfields=false

readproportion=0.5
updateproportion=0.5
scanproportion=0
insertproportion=0

requestdistribution=zipfian
Creating a new file that changes any of these values will produce a new workload with different characteristics. The set of properties that can be specified is here.

Option 2: new java class

The workload java class will be created by the YCSB Client at runtime, and will use an instance of the DB interface layer to generate the actual operations against the database. Thus, the java class only needs to decide (based on settings in the parameter file) what records to create for the data set, and what reads, updates etc. to generate for the transaction phase. The YCSB Client will take care of creating the workload java class, passing it to a worker thread for executing, deciding how many records to create or how many operations to execute, and measuring the resulting performance.

If the CoreWorkload (or some other existing package) does not have the ability to generate the workload you desire, you can create a new workload java class. This is done using the following steps:

Step 1. Extend com.yahoo.ycsb.Workload

The base class of all workload classes is com.yahoo.ycsb.Workload. This is an abstract class, so you create a new workload that extends this base class. Your class must have a public no-argument constructor, because the workload will be created in a factory using the no-argument constructor. The YCSB Client will create one Workload object for each worker thread, so if you run the Client with multiple threads, multiple workload objects will be created.

Step 2. Write code to initialize your workload class

The parameter fill will be passed to the workload object after the constructor has been called, so if you are using any parameter properties, you must use them to initialize your workload using either the init() or initThread() methods.
  • init() - called once for all workload instances. Used to initialize any objects shared by all threads.
  • initThread() - called once per workload instance in the context of the worker thread. Used to initialize any objects specific to a single Workload instance and single worker thread.
In either case, you can access the parameter properties using the Properties object passed in to both methods. These properties will include all properties defined in any property file passed to the YCSB Client or defined on the client command line.

Step 3. Write any cleanup code

The cleanup() method is called once for all workload instances, after the workload has completed.

Step 4. Define the records to be inserted

The YCSB Client will call the doInsert() method once for each record to be inserted into the database. So you should implement this method to create and insert a single record. The DB object you can use to perform the insert will be passed to the doInsert() method.

Step 5. Define the transactions

The YCSB Client will call the doTransaction() method once for every transaction that is to be executed. So you should implement this method to execute a single transaction, using the DB object passed in to access the database. Your implementation of this method can choose between different types of transactions, and can make multiple calls to the DB interface layer. However, each invocation of the method should be a logical transaction. In particular, when you run the client, you'll specify the number of operations to execute; if you request 1000 operations then doTransaction() will be executed 1000 times.

Note that you do not have to do any throttling of your transactions (or record insertions) to achieve the target throughput. The YCSB Client will do the throttling for you.

Note also that it is allowable to insert records inside the doTransaction() method. You might do this if you wish the database to grow during the workload. In this case, the initial dataset will be constructed using calls to the doInsert() method, while additional records would be inserted using calls to the doTransaction() method.

Step 6 - Measure latency, if necessary

The YCSB client will automatically measure the latency and throughput of database operations, even for workloads that you define. However, the client will only measure the latency of individual calls to the database, not of more complex transactions. Consider for example a workload that reads a record, modifies it, and writes the changes back to the database. The YCSB client will automatically measure the latency of the read operation to the database; and separately will automatically measure the latency of the update operation. However, if you would like to measure the latency of the entire read-modify-write transaction, you will need to add an additional timing step to your code.

Measurements are gathered using the Measurements.measure() call. There is a singleton instance of Measurements, which can be obtained using the Measurements.getMeasurements() static method. For each metric you are measuring, you need to assign a string tag; this tag will label the resulting average, min, max, histogram etc. measurements output by the tool at the end of the workload. For example, consider the following code: 

long st=System.currentTimeMillis();
db.read(TABLENAME,keyname,fields,new HashMap());
db.update(TABLENAME,keyname,values);
long en=System.currentTimeMillis();
Measurements.getMeasurements().measure("READ-MODIFY-WRITE", (int)(en-st));
In this code, the calls to System.currentTimeMillis() are used to time the read and write transaction. Then, the call to measure() reports the latency to the measurement component.

Using this pattern, your custom measurements will be gathered and aggregated using the same mechanism that is used to gather measurements for individual READ, UPDATE etc. operations.

Step 7 - Use it with the YCSB Client

Make sure that the classes for your implementation (or a jar containing those classes) are available on your CLASSPATH, as well as any libraries/jar files used by your implementation. Now, when you run the YCSB Client, specify the "workload" property to provide the fully qualified classname of your DB class. For example: 
workload=com.foo.YourWorkloadClass
YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. ================================================ FILE: dynamodb/README.md ================================================ # DynamoDB Binding http://aws.amazon.com/documentation/dynamodb/ ## Configure YCSB_HOME - YCSB home directory DYNAMODB_HOME - Amazon DynamoDB package files Please refer to https://github.com/brianfrankcooper/YCSB/wiki/Using-the-Database-Libraries for more information on setup. # Benchmark $YCSB_HOME/bin/ycsb load dynamodb -P workloads/workloada -P dynamodb.properties $YCSB_HOME/bin/ycsb run dynamodb -P workloads/workloada -P dynamodb.properties # Properties $DYNAMODB_HOME/conf/dynamodb.properties $DYNAMODB_HOME/conf/AWSCredentials.properties # FAQs * Why is the recommended workload distribution set to 'uniform'? This is to conform with the best practices for using DynamoDB - uniform, evenly distributed workload is the recommended pattern for scaling and getting predictable performance out of DynamoDB For more information refer to http://docs.amazonwebservices.com/amazondynamodb/latest/developerguide/BestPractices.html * How does workload size affect provisioned throughput? The default payload size requires double the provisioned throughput to execute the workload. This translates to double the provisioned throughput cost for testing. The default item size in YCSB are 1000 bytes plus metadata overhead, which makes the item exceed 1024 bytes. DynamoDB charges one capacity unit per 1024 bytes for read or writes. An item that is greater than 1024 bytes but less than or equal to 2048 bytes would cost 2 capacity units. With the change in payload size, each request would cost 1 capacity unit as opposed to 2, saving the cost of running the benchmark. For more information refer to http://docs.amazonwebservices.com/amazondynamodb/latest/developerguide/WorkingWithDDTables.html * How do you know if DynamoDB throttling is affecting benchmarking? Monitor CloudWatch for ThrottledRequests and if ThrottledRequests is greater than zero, either increase the DynamoDB table provisioned throughput or reduce YCSB throughput by reducing YCSB target throughput, adjusting the number of YCSB client threads, or combination of both. For more information please refer to https://github.com/brianfrankcooper/YCSB/blob/master/doc/tipsfaq.html When requests are throttled, latency measurements by YCSB can increase. Please refer to http://aws.amazon.com/dynamodb/faqs/ for more information. Please refer to Amazon DynamoDB docs here: http://aws.amazon.com/documentation/dynamodb/ ================================================ FILE: dynamodb/conf/AWSCredentials.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Fill in your AWS Access Key ID and Secret Access Key # http://aws.amazon.com/security-credentials #accessKey = #secretKey = ================================================ FILE: dynamodb/conf/dynamodb.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Sample property file for Amazon DynamoDB database client ## Mandatory parameters # AWS credentials associated with your aws account. #dynamodb.awsCredentialsFile = # Primarykey of table 'usertable' #dynamodb.primaryKey = # If you set dynamodb.primaryKeyType to HASH_AND_RANGE, you must specify the # hash key name of your primary key here. (see documentation below for details) #dynamodb.hashKeyName = ## Optional parameters # The property "primaryKeyType" below specifies the type of primary key # you have setup for the test table. There are two choices: # - HASH (default) # - HASH_AND_RANGE # # When testing the DB in HASH mode (which is the default), your table's # primary key must be of the "HASH" key type, and the name of the primary key # is specified via the dynamodb.primaryKey property. In this mode, all # keys from YCSB are hashed across multiple hash partitions and # performance of individual operations are good. However, query across # multiple items is eventually consistent in this mode and relies on the # global secondary index. # # # When testing the DB in HASH_AND_RANGE mode, your table's primary key must be # of the "HASH_AND_RANGE" key type. You need to specify the name of the # hash key via the "dynamodb.hashKeyName" property and you also need to # specify the name of the range key via the "dynamodb.primaryKey" property. # In this mode, keys supplied by YCSB will be used as the range part of # the primary key and the hash part of the primary key will have a fixed value. # Optionally you can designate the value used in the hash part of the primary # key via the dynamodb.hashKeyValue. # # The purpose of the HASH_AND_RANGE mode is to benchmark the performance # characteristics of a single logical hash partition. This is useful because # so far the only practical way to do strongly consistent query is to do it # in a single hash partition (Whole table scan can be consistent but it becomes # less practical when the table is really large). Therefore, for users who # really want to have strongly consistent query, it's important for them to # know the performance capabilities of a single logical hash partition so # they can plan their application accordingly. #dynamodb.primaryKeyType = HASH #Optionally you can specify a value for the hash part of the primary key #when testing in HASH_AND_RANG mode. #dynamodb.hashKeyValue = # Endpoint to connect to.For best latency, it is recommended # to choose the endpoint which is closer to the client. # Default is us-east-1 #dynamodb.endpoint = http://dynamodb.us-east-1.amazonaws.com # Strongly recommended to set to uniform.Refer FAQs in README #requestdistribution = uniform # Enable/disable debug messages.Defaults to false # "true" or "false" #dynamodb.debug = false # Maximum number of concurrent connections #dynamodb.connectMax = 50 # Read consistency.Consistent reads are expensive and consume twice # as many resources as eventually consistent reads. Defaults to false. # "true" or "false" #dynamodb.consistentReads = false # Workload size has implications on provisioned read and write # capacity units.Refer FAQs in README #fieldcount = 10 #fieldlength = 90 ================================================ FILE: dynamodb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent dynamodb-binding DynamoDB DB Binding com.amazonaws aws-java-sdk 1.10.48 log4j log4j 1.2.17 com.yahoo.ycsb core ${project.version} provided ================================================ FILE: dynamodb/src/main/java/com/yahoo/ycsb/db/DynamoDBClient.java ================================================ /* * Copyright 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved. * Copyright 2015-2016 YCSB Contributors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://aws.amazon.com/apache2.0 * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ package com.yahoo.ycsb.db; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.PropertiesCredentials; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.dynamodbv2.model.*; import com.yahoo.ycsb.*; import org.apache.log4j.Level; import org.apache.log4j.Logger; import java.io.File; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.Vector; /** * DynamoDB v1.10.48 client for YCSB. */ public class DynamoDBClient extends DB { /** * Defines the primary key type used in this particular DB instance. *

* By default, the primary key type is "HASH". Optionally, the user can * choose to use hash_and_range key type. See documentation in the * DynamoDB.Properties file for more details. */ private enum PrimaryKeyType { HASH, HASH_AND_RANGE } private AmazonDynamoDBClient dynamoDB; private String primaryKeyName; private PrimaryKeyType primaryKeyType = PrimaryKeyType.HASH; // If the user choose to use HASH_AND_RANGE as primary key type, then // the following two variables become relevant. See documentation in the // DynamoDB.Properties file for more details. private String hashKeyValue; private String hashKeyName; private boolean consistentRead = false; private String endpoint = "http://dynamodb.us-east-1.amazonaws.com"; private int maxConnects = 50; private static final Logger LOGGER = Logger.getLogger(DynamoDBClient.class); private static final Status CLIENT_ERROR = new Status("CLIENT_ERROR", "An error occurred on the client."); private static final String DEFAULT_HASH_KEY_VALUE = "YCSB_0"; @Override public void init() throws DBException { String debug = getProperties().getProperty("dynamodb.debug", null); if (null != debug && "true".equalsIgnoreCase(debug)) { LOGGER.setLevel(Level.DEBUG); } String configuredEndpoint = getProperties().getProperty("dynamodb.endpoint", null); String credentialsFile = getProperties().getProperty("dynamodb.awsCredentialsFile", null); String primaryKey = getProperties().getProperty("dynamodb.primaryKey", null); String primaryKeyTypeString = getProperties().getProperty("dynamodb.primaryKeyType", null); String consistentReads = getProperties().getProperty("dynamodb.consistentReads", null); String connectMax = getProperties().getProperty("dynamodb.connectMax", null); if (null != connectMax) { this.maxConnects = Integer.parseInt(connectMax); } if (null != consistentReads && "true".equalsIgnoreCase(consistentReads)) { this.consistentRead = true; } if (null != configuredEndpoint) { this.endpoint = configuredEndpoint; } if (null == primaryKey || primaryKey.length() < 1) { throw new DBException("Missing primary key attribute name, cannot continue"); } if (null != primaryKeyTypeString) { try { this.primaryKeyType = PrimaryKeyType.valueOf(primaryKeyTypeString.trim().toUpperCase()); } catch (IllegalArgumentException e) { throw new DBException("Invalid primary key mode specified: " + primaryKeyTypeString + ". Expecting HASH or HASH_AND_RANGE."); } } if (this.primaryKeyType == PrimaryKeyType.HASH_AND_RANGE) { // When the primary key type is HASH_AND_RANGE, keys used by YCSB // are range keys so we can benchmark performance of individual hash // partitions. In this case, the user must specify the hash key's name // and optionally can designate a value for the hash key. String configuredHashKeyName = getProperties().getProperty("dynamodb.hashKeyName", null); if (null == configuredHashKeyName || configuredHashKeyName.isEmpty()) { throw new DBException("Must specify a non-empty hash key name when the primary key type is HASH_AND_RANGE."); } this.hashKeyName = configuredHashKeyName; this.hashKeyValue = getProperties().getProperty("dynamodb.hashKeyValue", DEFAULT_HASH_KEY_VALUE); } try { AWSCredentials credentials = new PropertiesCredentials(new File(credentialsFile)); ClientConfiguration cconfig = new ClientConfiguration(); cconfig.setMaxConnections(maxConnects); dynamoDB = new AmazonDynamoDBClient(credentials, cconfig); dynamoDB.setEndpoint(this.endpoint); primaryKeyName = primaryKey; LOGGER.info("dynamodb connection created with " + this.endpoint); } catch (Exception e1) { LOGGER.error("DynamoDBClient.init(): Could not initialize DynamoDB client.", e1); } } @Override public Status read(String table, String key, Set fields, Map result) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("readkey: " + key + " from table: " + table); } GetItemRequest req = new GetItemRequest(table, createPrimaryKey(key)); req.setAttributesToGet(fields); req.setConsistentRead(consistentRead); GetItemResult res; try { res = dynamoDB.getItem(req); } catch (AmazonServiceException ex) { LOGGER.error(ex); return Status.ERROR; } catch (AmazonClientException ex) { LOGGER.error(ex); return CLIENT_ERROR; } if (null != res.getItem()) { result.putAll(extractResult(res.getItem())); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Result: " + res.toString()); } } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("scan " + recordcount + " records from key: " + startkey + " on table: " + table); } /* * on DynamoDB's scan, startkey is *exclusive* so we need to * getItem(startKey) and then use scan for the res */ GetItemRequest greq = new GetItemRequest(table, createPrimaryKey(startkey)); greq.setAttributesToGet(fields); GetItemResult gres; try { gres = dynamoDB.getItem(greq); } catch (AmazonServiceException ex) { LOGGER.error(ex); return Status.ERROR; } catch (AmazonClientException ex) { LOGGER.error(ex); return CLIENT_ERROR; } if (null != gres.getItem()) { result.add(extractResult(gres.getItem())); } int count = 1; // startKey is done, rest to go. Map startKey = createPrimaryKey(startkey); ScanRequest req = new ScanRequest(table); req.setAttributesToGet(fields); while (count < recordcount) { req.setExclusiveStartKey(startKey); req.setLimit(recordcount - count); ScanResult res; try { res = dynamoDB.scan(req); } catch (AmazonServiceException ex) { LOGGER.error(ex); return Status.ERROR; } catch (AmazonClientException ex) { LOGGER.error(ex); return CLIENT_ERROR; } count += res.getCount(); for (Map items : res.getItems()) { result.add(extractResult(items)); } startKey = res.getLastEvaluatedKey(); } return Status.OK; } @Override public Status update(String table, String key, Map values) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("updatekey: " + key + " from table: " + table); } Map attributes = new HashMap<>(values.size()); for (Entry val : values.entrySet()) { AttributeValue v = new AttributeValue(val.getValue().toString()); attributes.put(val.getKey(), new AttributeValueUpdate().withValue(v).withAction("PUT")); } UpdateItemRequest req = new UpdateItemRequest(table, createPrimaryKey(key), attributes); try { dynamoDB.updateItem(req); } catch (AmazonServiceException ex) { LOGGER.error(ex); return Status.ERROR; } catch (AmazonClientException ex) { LOGGER.error(ex); return CLIENT_ERROR; } return Status.OK; } @Override public Status insert(String table, String key, Map values) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("insertkey: " + primaryKeyName + "-" + key + " from table: " + table); } Map attributes = createAttributes(values); // adding primary key attributes.put(primaryKeyName, new AttributeValue(key)); if (primaryKeyType == PrimaryKeyType.HASH_AND_RANGE) { // If the primary key type is HASH_AND_RANGE, then what has been put // into the attributes map above is the range key part of the primary // key, we still need to put in the hash key part here. attributes.put(hashKeyName, new AttributeValue(hashKeyValue)); } PutItemRequest putItemRequest = new PutItemRequest(table, attributes); try { dynamoDB.putItem(putItemRequest); } catch (AmazonServiceException ex) { LOGGER.error(ex); return Status.ERROR; } catch (AmazonClientException ex) { LOGGER.error(ex); return CLIENT_ERROR; } return Status.OK; } @Override public Status delete(String table, String key) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("deletekey: " + key + " from table: " + table); } DeleteItemRequest req = new DeleteItemRequest(table, createPrimaryKey(key)); try { dynamoDB.deleteItem(req); } catch (AmazonServiceException ex) { LOGGER.error(ex); return Status.ERROR; } catch (AmazonClientException ex) { LOGGER.error(ex); return CLIENT_ERROR; } return Status.OK; } private static Map createAttributes(Map values) { Map attributes = new HashMap<>(values.size() + 1); for (Entry val : values.entrySet()) { attributes.put(val.getKey(), new AttributeValue(val.getValue().toString())); } return attributes; } private HashMap extractResult(Map item) { if (null == item) { return null; } HashMap rItems = new HashMap<>(item.size()); for (Entry attr : item.entrySet()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Result- key: %s, value: %s", attr.getKey(), attr.getValue())); } rItems.put(attr.getKey(), new StringByteIterator(attr.getValue().getS())); } return rItems; } private Map createPrimaryKey(String key) { Map k = new HashMap<>(); if (primaryKeyType == PrimaryKeyType.HASH) { k.put(primaryKeyName, new AttributeValue().withS(key)); } else if (primaryKeyType == PrimaryKeyType.HASH_AND_RANGE) { k.put(hashKeyName, new AttributeValue().withS(hashKeyValue)); k.put(primaryKeyName, new AttributeValue().withS(key)); } else { throw new RuntimeException("Assertion Error: impossible primary key type"); } return k; } } ================================================ FILE: dynamodb/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright 2015-2016 YCSB Contributors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for DynamoDB. */ package com.yahoo.ycsb.db; ================================================ FILE: dynamodb/src/main/resources/log4j.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. #define the console appender log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender # now define the layout for the appender log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n # now map our console appender as a root logger, means all log messages will go # to this appender log4j.rootLogger = INFO, consoleAppender ================================================ FILE: elasticsearch/README.md ================================================ ## Quick Start This section describes how to run YCSB on Elasticsearch running locally. ### 1. Set Up YCSB Clone the YCSB git repository and compile: git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ### 2. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load elasticsearch -s -P workloads/workloada -p path.home= Then, run the workload: ./bin/ycsb run elasticsearch -s -P workloads/workloada -p path.home= Note that the `` specified in each execution should be the same. The Elasticsearch binding has two modes of operation, embedded mode and remote mode. In embedded mode, the client creates an embedded instance of Elasticsearch that uses the specified `` to persist data between executions. In remote mode, the client will hit a standalone instance of Elasticsearch. To use remote mode, add the flags `-p es.remote=true` and specify a hosts list via `-p es.hosts.list=,...,`. ./bin/ycsb run elasticsearch -s -P workloads/workloada -p es.remote=true \ -p es.hosts.list=,...,` Note that `es.hosts.list` defaults to `localhost:9300`. For further configuration see below: ### Defaults Configuration The default setting for the Elasticsearch node that is created is as follows: cluster.name=es.ycsb.cluster es.index.key=es.ycsb es.number_of_shards=1 es.number_of_replicas=0 es.remote=false es.newdb=false es.hosts.list=localhost:9300 (only applies if es.remote=true) ### Custom Configuration If you wish to customize the settings used to create the Elasticsearch node you can created a new property file that contains your desired Elasticsearch node settings and pass it in via the parameter to 'bin/ycsb' script. Note that the default properties will be kept if you don't explicitly overwrite them. Assuming that we have a properties file named "myproperties.data" that contains custom Elasticsearch node configuration you can execute the following to pass it into the Elasticsearch client: ./bin/ycsb run elasticsearch -P workloads/workloada -P myproperties.data -s If you wish to change the default index name you can set the following property: es.index.key=my_index_key If you wish to run against a remote cluster you can set the following property: es.remote=true By default this will use localhost:9300 as a seed node to discover the cluster. You can also specify es.hosts.list=(\w+:\d+)+ (a comma-separated list of host/port pairs) to change this. ================================================ FILE: elasticsearch/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent elasticsearch-binding Elasticsearch Binding jar 2.4.0 net.java.dev.jna jna 4.1.0 com.yahoo.ycsb core ${project.version} provided org.elasticsearch elasticsearch ${elasticsearch-version} junit junit 4.12 test ================================================ FILE: elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticsearchClient.java ================================================ /** * Copyright (c) 2012 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static org.elasticsearch.common.settings.Settings.Builder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.node.NodeBuilder.nodeBuilder; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.node.Node; import org.elasticsearch.search.SearchHit; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * Elasticsearch client for YCSB framework. * *

* Default properties to set: *

*
    *
  • cluster.name = es.ycsb.cluster *
  • es.index.key = es.ycsb *
  • es.number_of_shards = 1 *
  • es.number_of_replicas = 0 *
*/ public class ElasticsearchClient extends DB { private static final String DEFAULT_CLUSTER_NAME = "es.ycsb.cluster"; private static final String DEFAULT_INDEX_KEY = "es.ycsb"; private static final String DEFAULT_REMOTE_HOST = "localhost:9300"; private static final int NUMBER_OF_SHARDS = 1; private static final int NUMBER_OF_REPLICAS = 0; private Node node; private Client client; private String indexKey; private Boolean remoteMode; /** * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void init() throws DBException { final Properties props = getProperties(); // Check if transport client needs to be used (To connect to multiple // elasticsearch nodes) remoteMode = Boolean.parseBoolean(props.getProperty("es.remote", "false")); final String pathHome = props.getProperty("path.home"); // when running in embedded mode, require path.home if (!remoteMode && (pathHome == null || pathHome.isEmpty())) { throw new IllegalArgumentException("path.home must be specified when running in embedded mode"); } this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY); int numberOfShards = parseIntegerProperty(props, "es.number_of_shards", NUMBER_OF_SHARDS); int numberOfReplicas = parseIntegerProperty(props, "es.number_of_replicas", NUMBER_OF_REPLICAS); Boolean newdb = Boolean.parseBoolean(props.getProperty("es.newdb", "false")); Builder settings = Settings.settingsBuilder() .put("cluster.name", DEFAULT_CLUSTER_NAME) .put("node.local", Boolean.toString(!remoteMode)) .put("path.home", pathHome); // if properties file contains elasticsearch user defined properties // add it to the settings file (will overwrite the defaults). settings.put(props); final String clusterName = settings.get("cluster.name"); System.err.println("Elasticsearch starting node = " + clusterName); System.err.println("Elasticsearch node path.home = " + settings.get("path.home")); System.err.println("Elasticsearch Remote Mode = " + remoteMode); // Remote mode support for connecting to remote elasticsearch cluster if (remoteMode) { settings.put("client.transport.sniff", true) .put("client.transport.ignore_cluster_name", false) .put("client.transport.ping_timeout", "30s") .put("client.transport.nodes_sampler_interval", "30s"); // Default it to localhost:9300 String[] nodeList = props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST).split(","); System.out.println("Elasticsearch Remote Hosts = " + props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST)); TransportClient tClient = TransportClient.builder().settings(settings).build(); for (String h : nodeList) { String[] nodes = h.split(":"); try { tClient.addTransportAddress(new InetSocketTransportAddress( InetAddress.getByName(nodes[0]), Integer.parseInt(nodes[1]) )); } catch (NumberFormatException e) { throw new IllegalArgumentException("Unable to parse port number.", e); } catch (UnknownHostException e) { throw new IllegalArgumentException("Unable to Identify host.", e); } } client = tClient; } else { // Start node only if transport client mode is disabled node = nodeBuilder().clusterName(clusterName).settings(settings).node(); node.start(); client = node.client(); } final boolean exists = client.admin().indices() .exists(Requests.indicesExistsRequest(indexKey)).actionGet() .isExists(); if (exists && newdb) { client.admin().indices().prepareDelete(indexKey).execute().actionGet(); } if (!exists || newdb) { client.admin().indices().create( new CreateIndexRequest(indexKey) .settings( Settings.builder() .put("index.number_of_shards", numberOfShards) .put("index.number_of_replicas", numberOfReplicas) .put("index.mapping._id.indexed", true) )).actionGet(); } client.admin().cluster().health(new ClusterHealthRequest().waitForGreenStatus()).actionGet(); } private int parseIntegerProperty(Properties properties, String key, int defaultValue) { String value = properties.getProperty(key); return value == null ? defaultValue : Integer.parseInt(value); } @Override public void cleanup() throws DBException { if (!remoteMode) { if (!node.isClosed()) { client.close(); node.close(); } } else { client.close(); } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public Status insert(String table, String key, Map values) { try { final XContentBuilder doc = jsonBuilder().startObject(); for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { doc.field(entry.getKey(), entry.getValue()); } doc.endObject(); client.prepareIndex(indexKey, table, key).setSource(doc).execute().actionGet(); return Status.OK; } catch (Exception e) { e.printStackTrace(); return Status.ERROR; } } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public Status delete(String table, String key) { try { DeleteResponse response = client.prepareDelete(indexKey, table, key).execute().actionGet(); if (response.isFound()) { return Status.OK; } else { return Status.NOT_FOUND; } } catch (Exception e) { e.printStackTrace(); return Status.ERROR; } } /** * Read a record from the database. Each field/value pair from the result will * be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public Status read(String table, String key, Set fields, Map result) { try { final GetResponse response = client.prepareGet(indexKey, table, key).execute().actionGet(); if (response.isExists()) { if (fields != null) { for (String field : fields) { result.put(field, new StringByteIterator( (String) response.getSource().get(field))); } } else { for (String field : response.getSource().keySet()) { result.put(field, new StringByteIterator( (String) response.getSource().get(field))); } } return Status.OK; } else { return Status.NOT_FOUND; } } catch (Exception e) { e.printStackTrace(); return Status.ERROR; } } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public Status update(String table, String key, Map values) { try { final GetResponse response = client.prepareGet(indexKey, table, key).execute().actionGet(); if (response.isExists()) { for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { response.getSource().put(entry.getKey(), entry.getValue()); } client.prepareIndex(indexKey, table, key).setSource(response.getSource()).execute().actionGet(); return Status.OK; } else { return Status.NOT_FOUND; } } catch (Exception e) { e.printStackTrace(); return Status.ERROR; } } /** * Perform a range scan for a set of records in the database. Each field/value * pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public Status scan( String table, String startkey, int recordcount, Set fields, Vector> result) { try { final RangeQueryBuilder rangeQuery = rangeQuery("_id").gte(startkey); final SearchResponse response = client.prepareSearch(indexKey) .setTypes(table) .setQuery(rangeQuery) .setSize(recordcount) .execute() .actionGet(); HashMap entry; for (SearchHit hit : response.getHits()) { entry = new HashMap<>(fields.size()); for (String field : fields) { entry.put(field, new StringByteIterator((String) hit.getSource().get(field))); } result.add(entry); } return Status.OK; } catch (Exception e) { e.printStackTrace(); return Status.ERROR; } } } ================================================ FILE: elasticsearch/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for * Elasticsearch. */ package com.yahoo.ycsb.db; ================================================ FILE: elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticsearchClientTest.java ================================================ /** * Copyright (c) 2012-2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /* * To change this template, choose Tools | Templates * and open the template in the editor. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import static org.junit.Assert.assertEquals; public class ElasticsearchClientTest { @ClassRule public final static TemporaryFolder temp = new TemporaryFolder(); private final static ElasticsearchClient instance = new ElasticsearchClient(); private final static HashMap MOCK_DATA; private final static String MOCK_TABLE = "MOCK_TABLE"; private final static String MOCK_KEY0 = "0"; private final static String MOCK_KEY1 = "1"; private final static String MOCK_KEY2 = "2"; static { MOCK_DATA = new HashMap<>(10); for (int i = 1; i <= 10; i++) { MOCK_DATA.put("field" + i, new StringByteIterator("value" + i)); } } @BeforeClass public static void setUpClass() throws DBException { final Properties props = new Properties(); props.put("path.home", temp.getRoot().toString()); instance.setProperties(props); instance.init(); } @AfterClass public static void tearDownClass() throws DBException { instance.cleanup(); } @Before public void setUp() { instance.insert(MOCK_TABLE, MOCK_KEY1, MOCK_DATA); instance.insert(MOCK_TABLE, MOCK_KEY2, MOCK_DATA); } @After public void tearDown() { instance.delete(MOCK_TABLE, MOCK_KEY1); instance.delete(MOCK_TABLE, MOCK_KEY2); } /** * Test of insert method, of class ElasticsearchClient. */ @Test public void testInsert() { Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA); assertEquals(Status.OK, result); } /** * Test of delete method, of class ElasticsearchClient. */ @Test public void testDelete() { Status result = instance.delete(MOCK_TABLE, MOCK_KEY1); assertEquals(Status.OK, result); } /** * Test of read method, of class ElasticsearchClient. */ @Test public void testRead() { Set fields = MOCK_DATA.keySet(); HashMap resultParam = new HashMap<>(10); Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam); assertEquals(Status.OK, result); } /** * Test of update method, of class ElasticsearchClient. */ @Test public void testUpdate() { int i; HashMap newValues = new HashMap<>(10); for (i = 1; i <= 10; i++) { newValues.put("field" + i, new StringByteIterator("newvalue" + i)); } Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues); assertEquals(Status.OK, result); //validate that the values changed HashMap resultParam = new HashMap<>(10); instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam); for (i = 1; i <= 10; i++) { assertEquals("newvalue" + i, resultParam.get("field" + i).toString()); } } /** * Test of scan method, of class ElasticsearchClient. */ @Test public void testScan() { int recordcount = 10; Set fields = MOCK_DATA.keySet(); Vector> resultParam = new Vector<>(10); Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, recordcount, fields, resultParam); assertEquals(Status.OK, result); } } ================================================ FILE: elasticsearch5/README.md ================================================ ## Quick Start This section describes how to run YCSB on Elasticsearch 5.x running locally. ### 1. Install and Start Elasticsearch [Download and install Elasticsearch][1]. When starting Elasticsearch, you should [configure][2] the cluster name to be `es.ycsb.cluster` (see below). ### 2. Set Up YCSB Clone the YCSB git repository and compile: git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ### 3. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load elasticsearch5 -s -P workloads/workloada Then, run the workload: ./bin/ycsb run elasticsearch5 -s -P workloads/workloada The Elasticsearch 5 binding requires a standalone instance of Elasticsearch. You must specify a hosts list for the transport client to connect to via `-p es.hosts.list=,...,`: ./bin/ycsb run elasticsearch5 -s -P workloads/workloada \ -p es.hosts.list=,...,` Note that `es.hosts.list` defaults to `localhost:9300`. For further configuration see below: ### Defaults Configuration The default setting for the Elasticsearch node that is created is as follows: es.setting.cluster.name=es.ycsb.cluster es.index.key=es.ycsb es.number_of_shards=1 es.number_of_replicas=0 es.new_index=false es.hosts.list=localhost:9300 ### Custom Configuration If you wish to customize the settings used to create the Elasticsearch node you can created a new property file that contains your desired Elasticsearch node settings and pass it in via the parameter to 'bin/ycsb' script. Note that the default properties will be kept if you don't explicitly overwrite them. Assuming that we have a properties file named "myproperties.data" that contains custom Elasticsearch node configuration you can execute the following to pass it into the Elasticsearch client: ./bin/ycsb run elasticsearch5 -P workloads/workloada -P myproperties.data -s If you wish to change the default index name you can set the following property: es.index.key=my_index_key By default this will use localhost:9300 as a seed node to discover the cluster. You can also specify es.hosts.list=(\w+:\d+)+ (a comma-separated list of host/port pairs) to change this. ### Configuring the transport client The `elasticsearch5` binding starts a transport client to connect to Elasticsearch using the transport protocol. You can pass arbitrary settings to this instance by using properties with the prefix `es.setting.` followed by any valid Elasticsearch setting. For example, assuming that you started your Elasticsearch node with the cluster name `my-elasticsearch-cluster`, you would need to configure the transport client to use the same cluster name via ./bin/ycsb run elasticsearch5 -P \ -p es.setting.cluster.name=my-elasticsearch-cluster ### Using the Elasticsearch low-level REST client The Elasticsearch 5 bindings also ship with an implementation that uses the low-level Elasticsearch REST client. The name of this binding is `elasticsearch-rest`. For example: ./bin/ycsb load elasticsearch5-rest -P workloads/workloada You can configure the hosts to connect to via the same `es.hosts.list` property used to configure the transport client in the `elasticsearch5` binding (note that by default you should use port 9200) [1]: https://www.elastic.co/guide/en/elasticsearch/reference/5.5/_installation.html [2]: https://www.elastic.co/guide/en/elasticsearch/reference/5.5/settings.html ================================================ FILE: elasticsearch5/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent true org.elasticsearch.distribution.zip 9400 9500 ${skipTests} org.apache.maven.plugins maven-dependency-plugin 2.10 integ-setup-dependencies pre-integration-test copy ${skipTests} ${elasticsearch.groupid} elasticsearch ${elasticsearch5-version} zip true ${project.build.directory}/integration-tests/binaries org.apache.maven.plugins maven-surefire-plugin 2.19 default-test none com.carrotsearch.randomizedtesting junit4-maven-plugin 2.3.3 unit-tests test junit4 true ${skipTests} **/*Test.class **/*$* integration-tests integration-test junit4 true ${skipTests} **/*IT.class **/*$* elasticsearch5-binding Elasticsearch 5.x Binding jar org.elasticsearch jna 4.4.0 com.yahoo.ycsb core ${project.version} provided org.elasticsearch.client transport ${elasticsearch5-version} org.elasticsearch.client rest ${elasticsearch5-version} org.apache.logging.log4j log4j-api 2.8.2 org.apache.logging.log4j log4j-core 2.8.2 junit junit 4.12 test jdk8-tests 1.8 com.github.alexcojocaru elasticsearch-maven-plugin 5.9 ${elasticsearch5-version} test 9200 9300 start-elasticsearch pre-integration-test runforked stop-elasticsearch post-integration-test stop false ================================================ FILE: elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/Elasticsearch5.java ================================================ /* * Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.elasticsearch5; import java.util.Properties; final class Elasticsearch5 { private Elasticsearch5() { } static final String KEY = "key"; static int parseIntegerProperty(final Properties properties, final String key, final int defaultValue) { final String value = properties.getProperty(key); return value == null ? defaultValue : Integer.parseInt(value); } } ================================================ FILE: elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClient.java ================================================ /* * Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.elasticsearch5; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.transport.client.PreBuiltTransportClient; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Vector; import static com.yahoo.ycsb.db.elasticsearch5.Elasticsearch5.KEY; import static com.yahoo.ycsb.db.elasticsearch5.Elasticsearch5.parseIntegerProperty; import static org.elasticsearch.common.settings.Settings.Builder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; /** * Elasticsearch client for YCSB framework. */ public class ElasticsearchClient extends DB { private static final String DEFAULT_CLUSTER_NAME = "es.ycsb.cluster"; private static final String DEFAULT_INDEX_KEY = "es.ycsb"; private static final String DEFAULT_REMOTE_HOST = "localhost:9300"; private static final int NUMBER_OF_SHARDS = 1; private static final int NUMBER_OF_REPLICAS = 0; private TransportClient client; private String indexKey; /** * * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void init() throws DBException { final Properties props = getProperties(); this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY); final int numberOfShards = parseIntegerProperty(props, "es.number_of_shards", NUMBER_OF_SHARDS); final int numberOfReplicas = parseIntegerProperty(props, "es.number_of_replicas", NUMBER_OF_REPLICAS); final Boolean newIndex = Boolean.parseBoolean(props.getProperty("es.new_index", "false")); final Builder settings = Settings.builder().put("cluster.name", DEFAULT_CLUSTER_NAME); // if properties file contains elasticsearch user defined properties // add it to the settings file (will overwrite the defaults). for (final Entry e : props.entrySet()) { if (e.getKey() instanceof String) { final String key = (String) e.getKey(); if (key.startsWith("es.setting.")) { settings.put(key.substring("es.setting.".length()), e.getValue()); } } } settings.put("client.transport.sniff", true) .put("client.transport.ignore_cluster_name", false) .put("client.transport.ping_timeout", "30s") .put("client.transport.nodes_sampler_interval", "30s"); // Default it to localhost:9300 final String[] nodeList = props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST).split(","); client = new PreBuiltTransportClient(settings.build()); for (String h : nodeList) { String[] nodes = h.split(":"); final InetAddress address; try { address = InetAddress.getByName(nodes[0]); } catch (UnknownHostException e) { throw new IllegalArgumentException("unable to identity host [" + nodes[0]+ "]", e); } final int port; try { port = Integer.parseInt(nodes[1]); } catch (final NumberFormatException e) { throw new IllegalArgumentException("unable to parse port [" + nodes[1] + "]", e); } client.addTransportAddress(new InetSocketTransportAddress(address, port)); } final boolean exists = client.admin().indices() .exists(Requests.indicesExistsRequest(indexKey)).actionGet() .isExists(); if (exists && newIndex) { client.admin().indices().prepareDelete(indexKey).get(); } if (!exists || newIndex) { client.admin().indices().create( new CreateIndexRequest(indexKey) .settings( Settings.builder() .put("index.number_of_shards", numberOfShards) .put("index.number_of_replicas", numberOfReplicas) )).actionGet(); } client.admin().cluster().health(new ClusterHealthRequest().waitForGreenStatus()).actionGet(); } @Override public void cleanup() throws DBException { if (client != null) { client.close(); client = null; } } private volatile boolean isRefreshNeeded = false; @Override public Status insert(String table, String key, Map values) { try (XContentBuilder doc = jsonBuilder()) { doc.startObject(); for (final Entry entry : StringByteIterator.getStringMap(values).entrySet()) { doc.field(entry.getKey(), entry.getValue()); } doc.field(KEY, key); doc.endObject(); final IndexResponse indexResponse = client.prepareIndex(indexKey, table).setSource(doc).get(); if (indexResponse.getResult() != DocWriteResponse.Result.CREATED) { return Status.ERROR; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status delete(final String table, final String key) { try { final SearchResponse searchResponse = search(table, key); if (searchResponse.getHits().totalHits == 0) { return Status.NOT_FOUND; } final String id = searchResponse.getHits().getAt(0).getId(); final DeleteResponse deleteResponse = client.prepareDelete(indexKey, table, id).get(); if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { return Status.NOT_FOUND; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status read( final String table, final String key, final Set fields, final Map result) { try { final SearchResponse searchResponse = search(table, key); if (searchResponse.getHits().totalHits == 0) { return Status.NOT_FOUND; } final SearchHit hit = searchResponse.getHits().getAt(0); if (fields != null) { for (final String field : fields) { result.put(field, new StringByteIterator((String) hit.getSource().get(field))); } } else { for (final Map.Entry e : hit.getSource().entrySet()) { if (KEY.equals(e.getKey())) { continue; } result.put(e.getKey(), new StringByteIterator((String) e.getValue())); } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status update(final String table, final String key, final Map values) { try { final SearchResponse response = search(table, key); if (response.getHits().totalHits == 0) { return Status.NOT_FOUND; } final SearchHit hit = response.getHits().getAt(0); for (final Entry entry : StringByteIterator.getStringMap(values).entrySet()) { hit.getSource().put(entry.getKey(), entry.getValue()); } final IndexResponse indexResponse = client.prepareIndex(indexKey, table, hit.getId()).setSource(hit.getSource()).get(); if (indexResponse.getResult() != DocWriteResponse.Result.UPDATED) { return Status.ERROR; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status scan( final String table, final String startkey, final int recordcount, final Set fields, final Vector> result) { try { refreshIfNeeded(); final RangeQueryBuilder query = new RangeQueryBuilder(KEY).gte(startkey); final SearchResponse response = client.prepareSearch(indexKey).setQuery(query).setSize(recordcount).get(); for (final SearchHit hit : response.getHits()) { final HashMap entry; if (fields != null) { entry = new HashMap<>(fields.size()); for (final String field : fields) { entry.put(field, new StringByteIterator((String) hit.getSource().get(field))); } } else { entry = new HashMap<>(hit.getSource().size()); for (final Map.Entry field : hit.getSource().entrySet()) { if (KEY.equals(field.getKey())) { continue; } entry.put(field.getKey(), new StringByteIterator((String) field.getValue())); } } result.add(entry); } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } private void refreshIfNeeded() { if (isRefreshNeeded) { final boolean refresh; synchronized (this) { if (isRefreshNeeded) { refresh = true; isRefreshNeeded = false; } else { refresh = false; } } if (refresh) { client.admin().indices().refresh(new RefreshRequest()).actionGet(); } } } private SearchResponse search(final String table, final String key) { refreshIfNeeded(); return client.prepareSearch(indexKey).setTypes(table).setQuery(new TermQueryBuilder(KEY, key)).get(); } } ================================================ FILE: elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchRestClient.java ================================================ /* * Copyright (c) 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.elasticsearch5; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpStatus; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import org.apache.http.nio.entity.NStringEntity; import org.codehaus.jackson.map.ObjectMapper; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import static com.yahoo.ycsb.db.elasticsearch5.Elasticsearch5.KEY; import static com.yahoo.ycsb.db.elasticsearch5.Elasticsearch5.parseIntegerProperty; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; /** * Elasticsearch REST client for YCSB framework. */ public class ElasticsearchRestClient extends DB { private static final String DEFAULT_INDEX_KEY = "es.ycsb"; private static final String DEFAULT_REMOTE_HOST = "localhost:9200"; private static final int NUMBER_OF_SHARDS = 1; private static final int NUMBER_OF_REPLICAS = 0; private RestClient restClient; private String indexKey; /** * * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void init() throws DBException { final Properties props = getProperties(); this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY); final int numberOfShards = parseIntegerProperty(props, "es.number_of_shards", NUMBER_OF_SHARDS); final int numberOfReplicas = parseIntegerProperty(props, "es.number_of_replicas", NUMBER_OF_REPLICAS); final Boolean newIndex = Boolean.parseBoolean(props.getProperty("es.new_index", "false")); final String[] nodeList = props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST).split(","); final List esHttpHosts = new ArrayList<>(nodeList.length); for (String h : nodeList) { String[] nodes = h.split(":"); esHttpHosts.add(new HttpHost(nodes[0], Integer.valueOf(nodes[1]), "http")); } restClient = RestClient.builder(esHttpHosts.toArray(new HttpHost[esHttpHosts.size()])).build(); final Response existsResponse = performRequest(restClient, "HEAD", "/" + indexKey); final boolean exists = existsResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK; if (exists && newIndex) { final Response deleteResponse = performRequest(restClient, "DELETE", "/" + indexKey); final int statusCode = deleteResponse.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { throw new DBException("delete [" + indexKey + "] failed with status [" + statusCode + "]"); } } if (!exists || newIndex) { try (XContentBuilder builder = jsonBuilder()) { builder.startObject(); builder.startObject("settings"); builder.field("index.number_of_shards", numberOfShards); builder.field("index.number_of_replicas", numberOfReplicas); builder.endObject(); builder.endObject(); final Map params = emptyMap(); final StringEntity entity = new StringEntity(builder.string()); final Response createResponse = performRequest(restClient, "PUT", "/" + indexKey, params, entity); final int statusCode = createResponse.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { throw new DBException("create [" + indexKey + "] failed with status [" + statusCode + "]"); } } catch (final IOException e) { throw new DBException(e); } } final Map params = Collections.singletonMap("wait_for_status", "green"); final Response healthResponse = performRequest(restClient, "GET", "/_cluster/health/" + indexKey, params); final int healthStatusCode = healthResponse.getStatusLine().getStatusCode(); if (healthStatusCode != HttpStatus.SC_OK) { throw new DBException("cluster health [" + indexKey + "] failed with status [" + healthStatusCode + "]"); } } private static Response performRequest( final RestClient restClient, final String method, final String endpoint) throws DBException { final Map params = emptyMap(); return performRequest(restClient, method, endpoint, params); } private static Response performRequest( final RestClient restClient, final String method, final String endpoint, final Map params) throws DBException { return performRequest(restClient, method, endpoint, params, null); } private static final Header[] EMPTY_HEADERS = new Header[0]; private static Response performRequest( final RestClient restClient, final String method, final String endpoint, final Map params, final HttpEntity entity) throws DBException { try { final Header[] headers; if (entity != null) { headers = new Header[]{new BasicHeader("content-type", ContentType.APPLICATION_JSON.toString())}; } else { headers = EMPTY_HEADERS; } return restClient.performRequest( method, endpoint, params, entity, headers); } catch (final IOException e) { e.printStackTrace(); throw new DBException(e); } } @Override public void cleanup() throws DBException { if (restClient != null) { try { restClient.close(); restClient = null; } catch (final IOException e) { throw new DBException(e); } } } private volatile boolean isRefreshNeeded = false; @Override public Status insert(final String table, final String key, final Map values) { try { final Map data = StringByteIterator.getStringMap(values); data.put(KEY, key); final Response response = restClient.performRequest( "POST", "/" + indexKey + "/" + table + "/", Collections.emptyMap(), new NStringEntity(new ObjectMapper().writeValueAsString(data), ContentType.APPLICATION_JSON)); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { return Status.ERROR; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status delete(final String table, final String key) { try { final Response searchResponse = search(table, key); final int statusCode = searchResponse.getStatusLine().getStatusCode(); if (statusCode == HttpStatus.SC_NOT_FOUND) { return Status.NOT_FOUND; } else if (statusCode != HttpStatus.SC_OK) { return Status.ERROR; } final Map map = map(searchResponse); @SuppressWarnings("unchecked") final Map hits = (Map)map.get("hits"); final int total = (int)hits.get("total"); if (total == 0) { return Status.NOT_FOUND; } @SuppressWarnings("unchecked") final Map hit = (Map)((List)hits.get("hits")).get(0); final Response deleteResponse = restClient.performRequest("DELETE", "/" + indexKey + "/" + table + "/" + hit.get("_id")); if (deleteResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { return Status.ERROR; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status read( final String table, final String key, final Set fields, final Map result) { try { final Response searchResponse = search(table, key); final int statusCode = searchResponse.getStatusLine().getStatusCode(); if (statusCode == 404) { return Status.NOT_FOUND; } else if (statusCode != HttpStatus.SC_OK) { return Status.ERROR; } final Map map = map(searchResponse); @SuppressWarnings("unchecked") final Map hits = (Map)map.get("hits"); final int total = (int)hits.get("total"); if (total == 0) { return Status.NOT_FOUND; } @SuppressWarnings("unchecked") final Map hit = (Map)((List)hits.get("hits")).get(0); @SuppressWarnings("unchecked") final Map source = (Map)hit.get("_source"); if (fields != null) { for (final String field : fields) { result.put(field, new StringByteIterator((String) source.get(field))); } } else { for (final Map.Entry e : source.entrySet()) { if (KEY.equals(e.getKey())) { continue; } result.put(e.getKey(), new StringByteIterator((String) e.getValue())); } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status update(final String table, final String key, final Map values) { try { final Response searchResponse = search(table, key); final int statusCode = searchResponse.getStatusLine().getStatusCode(); if (statusCode == 404) { return Status.NOT_FOUND; } else if (statusCode != HttpStatus.SC_OK) { return Status.ERROR; } final Map map = map(searchResponse); @SuppressWarnings("unchecked") final Map hits = (Map) map.get("hits"); final int total = (int) hits.get("total"); if (total == 0) { return Status.NOT_FOUND; } @SuppressWarnings("unchecked") final Map hit = (Map) ((List) hits.get("hits")).get(0); @SuppressWarnings("unchecked") final Map source = (Map) hit.get("_source"); for (final Map.Entry entry : StringByteIterator.getStringMap(values).entrySet()) { source.put(entry.getKey(), entry.getValue()); } final Map params = emptyMap(); final Response response = restClient.performRequest( "PUT", "/" + indexKey + "/" + table + "/" + hit.get("_id"), params, new NStringEntity(new ObjectMapper().writeValueAsString(source), ContentType.APPLICATION_JSON)); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { return Status.ERROR; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } @Override public Status scan( final String table, final String startkey, final int recordcount, final Set fields, final Vector> result) { try { final Response response; try (XContentBuilder builder = jsonBuilder()) { builder.startObject(); builder.startObject("query"); builder.startObject("range"); builder.startObject(KEY); builder.field("gte", startkey); builder.endObject(); builder.endObject(); builder.endObject(); builder.field("size", recordcount); builder.endObject(); response = search(table, builder); @SuppressWarnings("unchecked") final Map map = map(response); @SuppressWarnings("unchecked") final Map hits = (Map)map.get("hits"); @SuppressWarnings("unchecked") final List> list = (List>) hits.get("hits"); for (final Map hit : list) { @SuppressWarnings("unchecked") final Map source = (Map)hit.get("_source"); final HashMap entry; if (fields != null) { entry = new HashMap<>(fields.size()); for (final String field : fields) { entry.put(field, new StringByteIterator((String) source.get(field))); } } else { entry = new HashMap<>(hit.size()); for (final Map.Entry field : source.entrySet()) { if (KEY.equals(field.getKey())) { continue; } entry.put(field.getKey(), new StringByteIterator((String) field.getValue())); } } result.add(entry); } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } private void refreshIfNeeded() throws IOException { if (isRefreshNeeded) { final boolean refresh; synchronized (this) { if (isRefreshNeeded) { refresh = true; isRefreshNeeded = false; } else { refresh = false; } } if (refresh) { restClient.performRequest("POST", "/" + indexKey + "/_refresh"); } } } private Response search(final String table, final String key) throws IOException { try (XContentBuilder builder = jsonBuilder()) { builder.startObject(); builder.startObject("query"); builder.startObject("term"); builder.field(KEY, key); builder.endObject(); builder.endObject(); builder.endObject(); return search(table, builder); } } private Response search(final String table, final XContentBuilder builder) throws IOException { refreshIfNeeded(); final Map params = emptyMap(); final StringEntity entity = new StringEntity(builder.string()); final Header header = new BasicHeader("content-type", ContentType.APPLICATION_JSON.toString()); return restClient.performRequest("GET", "/" + indexKey + "/" + table + "/_search", params, entity, header); } private Map map(final Response response) throws IOException { try (InputStream is = response.getEntity().getContent()) { final ObjectMapper mapper = new ObjectMapper(); @SuppressWarnings("unchecked") final Map map = mapper.readValue(is, Map.class); return map; } } } ================================================ FILE: elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/package-info.java ================================================ /* * Copyright (c) 2017 YCSB Contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for * Elasticsearch. */ package com.yahoo.ycsb.db.elasticsearch5; ================================================ FILE: elasticsearch5/src/main/resources/log4j2.properties ================================================ appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n appender.console.targetStr = SYSTEM_ERR rootLogger.level = info rootLogger.appenderRef.console.ref = console ================================================ FILE: elasticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClientIT.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.elasticsearch5; import com.yahoo.ycsb.DB; public class ElasticsearchClientIT extends ElasticsearchIntegTestBase { @Override DB newDB() { return new ElasticsearchClient(); } } ================================================ FILE: elasticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchIntegTestBase.java ================================================ package com.yahoo.ycsb.db.elasticsearch5; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Client; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import static org.junit.Assert.assertEquals; public abstract class ElasticsearchIntegTestBase { private DB db; abstract DB newDB(); private final static HashMap MOCK_DATA; private final static String MOCK_TABLE = "MOCK_TABLE"; static { MOCK_DATA = new HashMap<>(10); for (int i = 1; i <= 10; i++) { MOCK_DATA.put("field" + i, new StringByteIterator("value" + i)); } } @Before public void setUp() throws DBException { final Properties props = new Properties(); props.put("es.new_index", "true"); props.put("es.setting.cluster.name", "test"); db = newDB(); db.setProperties(props); db.init(); for (int i = 0; i < 16; i++) { db.insert(MOCK_TABLE, Integer.toString(i), MOCK_DATA); } } @After public void tearDown() throws DBException { db.cleanup(); } @Test public void testInsert() { final Status result = db.insert(MOCK_TABLE, "0", MOCK_DATA); assertEquals(Status.OK, result); } /** * Test of delete method, of class ElasticsearchClient. */ @Test public void testDelete() { final Status result = db.delete(MOCK_TABLE, "1"); assertEquals(Status.OK, result); } /** * Test of read method, of class ElasticsearchClient. */ @Test public void testRead() { final Set fields = MOCK_DATA.keySet(); final HashMap resultParam = new HashMap<>(10); final Status result = db.read(MOCK_TABLE, "1", fields, resultParam); assertEquals(Status.OK, result); } /** * Test of update method, of class ElasticsearchClient. */ @Test public void testUpdate() { final HashMap newValues = new HashMap<>(10); for (int i = 1; i <= 10; i++) { newValues.put("field" + i, new StringByteIterator("newvalue" + i)); } final Status updateResult = db.update(MOCK_TABLE, "1", newValues); assertEquals(Status.OK, updateResult); // validate that the values changed final HashMap resultParam = new HashMap<>(10); final Status readResult = db.read(MOCK_TABLE, "1", MOCK_DATA.keySet(), resultParam); assertEquals(Status.OK, readResult); for (int i = 1; i <= 10; i++) { assertEquals("newvalue" + i, resultParam.get("field" + i).toString()); } } /** * Test of scan method, of class ElasticsearchClient. */ @Test public void testScan() { final int recordcount = 10; final Set fields = MOCK_DATA.keySet(); final Vector> resultParam = new Vector<>(10); final Status result = db.scan(MOCK_TABLE, "1", recordcount, fields, resultParam); assertEquals(Status.OK, result); assertEquals(10, resultParam.size()); } } ================================================ FILE: elasticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchRestClientIT.java ================================================ /** * Copyright (c) 2017 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.elasticsearch5; import com.yahoo.ycsb.DB; public class ElasticsearchRestClientIT extends ElasticsearchIntegTestBase { @Override DB newDB() { return new ElasticsearchRestClient(); } } ================================================ FILE: geode/README.md ================================================ ## Quick Start This section describes how to run YCSB on Apache Geode (incubating). ### Get Apache Geode You can download Geode from http://geode.incubator.apache.org/releases/ #### Start Geode Cluster Use the Geode shell (gfsh) to start the cluster. You will need to start at-least one locator which is a member discovery service and one or more Geode servers. Launch gfsh: ``` $ cd $GEODE_HOME $ ./bin/gfsh ``` Start a locator and two servers: ``` gfsh> start locator --name=locator1 gfsh> configure pdx --read-serialized=true gfsh> start server --name=server1 --server-port=40404 gfsh> start server --name=server2 --server-port=40405 ``` Create the "usertable" region required by YCSB driver: ``` gfsh>create region --name=usertable --type=PARTITION ``` gfsh has tab autocompletion, so you can play around with various options. ### Start YCSB workload From your YCSB directory, you can run the ycsb workload as follows ``` ./bin/ycsb load geode -P workloads/workloada -p geode.locator=host[port] ``` (default port of locator is 10334). In the default mode, ycsb geode driver will connect as a client to the geode cluster. To make the ycsb driver a peer member of the distributed system use the property `-p geode.topology=p2p -p geode.locator=host[port]` Note: For update workloads, please use the property `-p writeallfields=true` ================================================ FILE: geode/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent geode-binding Geode DB Binding jar org.apache.geode geode-core ${geode.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: geode/src/main/java/com/yahoo/ycsb/db/GeodeClient.java ================================================ /** * Copyright (c) 2013 - 2016 YCSB Contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import org.apache.geode.cache.*; import org.apache.geode.cache.client.ClientCache; import org.apache.geode.cache.client.ClientCacheFactory; import org.apache.geode.cache.client.ClientRegionFactory; import org.apache.geode.cache.client.ClientRegionShortcut; import org.apache.geode.internal.admin.remote.DistributionLocatorId; import org.apache.geode.internal.cache.GemFireCacheImpl; import org.apache.geode.pdx.JSONFormatter; import org.apache.geode.pdx.PdxInstance; import org.apache.geode.pdx.PdxInstanceFactory; import com.yahoo.ycsb.*; import java.util.*; /** * Apache Geode (incubating) client for the YCSB benchmark.
*

By default acts as a Geode client and tries to connect * to Geode cache server running on localhost with default * cache server port. Hostname and port of a Geode cacheServer * can be provided using geode.serverport=port and * geode.serverhost=host properties on YCSB command line. * A locator may also be used for discovering a cacheServer * by using the property geode.locator=host[port]

*

*

To run this client in a peer-to-peer topology with other Geode * nodes, use the property geode.topology=p2p. Running * in p2p mode will enable embedded caching in this client.

*

*

YCSB by default does its operations against "usertable". When running * as a client this is a ClientRegionShortcut.PROXY region, * when running in p2p mode it is a RegionShortcut.PARTITION * region. A cache.xml defining "usertable" region can be placed in the * working directory to override these region definitions.

*/ public class GeodeClient extends DB { /** * property name of the port where Geode server is listening for connections. */ private static final String SERVERPORT_PROPERTY_NAME = "geode.serverport"; /** * property name of the host where Geode server is running. */ private static final String SERVERHOST_PROPERTY_NAME = "geode.serverhost"; /** * default value of {@link #SERVERHOST_PROPERTY_NAME}. */ private static final String SERVERHOST_PROPERTY_DEFAULT = "localhost"; /** * property name to specify a Geode locator. This property can be used in both * client server and p2p topology */ private static final String LOCATOR_PROPERTY_NAME = "geode.locator"; /** * property name to specify Geode topology. */ private static final String TOPOLOGY_PROPERTY_NAME = "geode.topology"; /** * value of {@value #TOPOLOGY_PROPERTY_NAME} when peer to peer topology should be used. * (client-server topology is default) */ private static final String TOPOLOGY_P2P_VALUE = "p2p"; private GemFireCache cache; /** * true if ycsb client runs as a client to a Geode cache server. */ private boolean isClient; @Override public void init() throws DBException { Properties props = getProperties(); // hostName where Geode cacheServer is running String serverHost = null; // port of Geode cacheServer int serverPort = 0; String locatorStr = null; if (props != null && !props.isEmpty()) { String serverPortStr = props.getProperty(SERVERPORT_PROPERTY_NAME); if (serverPortStr != null) { serverPort = Integer.parseInt(serverPortStr); } serverHost = props.getProperty(SERVERHOST_PROPERTY_NAME, SERVERHOST_PROPERTY_DEFAULT); locatorStr = props.getProperty(LOCATOR_PROPERTY_NAME); String topology = props.getProperty(TOPOLOGY_PROPERTY_NAME); if (topology != null && topology.equals(TOPOLOGY_P2P_VALUE)) { CacheFactory cf = new CacheFactory(); if (locatorStr != null) { cf.set("locators", locatorStr); } cache = cf.create(); isClient = false; return; } } isClient = true; DistributionLocatorId locator = null; if (locatorStr != null) { locator = new DistributionLocatorId(locatorStr); } ClientCacheFactory ccf = new ClientCacheFactory(); ccf.setPdxReadSerialized(true); if (serverPort != 0) { ccf.addPoolServer(serverHost, serverPort); } else if (locator != null) { ccf.addPoolLocator(locator.getHost().getCanonicalHostName(), locator.getPort()); } cache = ccf.create(); } @Override public Status read(String table, String key, Set fields, Map result) { Region r = getRegion(table); PdxInstance val = r.get(key); if (val != null) { if (fields == null) { for (String fieldName : val.getFieldNames()) { result.put(fieldName, new ByteArrayByteIterator((byte[]) val.getField(fieldName))); } } else { for (String field : fields) { result.put(field, new ByteArrayByteIterator((byte[]) val.getField(field))); } } return Status.OK; } return Status.ERROR; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // Geode does not support scan return Status.ERROR; } @Override public Status update(String table, String key, Map values) { getRegion(table).put(key, convertToBytearrayMap(values)); return Status.OK; } @Override public Status insert(String table, String key, Map values) { getRegion(table).put(key, convertToBytearrayMap(values)); return Status.OK; } @Override public Status delete(String table, String key) { getRegion(table).destroy(key); return Status.OK; } private PdxInstance convertToBytearrayMap(Map values) { GemFireCacheImpl gci = (GemFireCacheImpl) CacheFactory.getAnyInstance(); PdxInstanceFactory pdxInstanceFactory = gci.createPdxInstanceFactory(JSONFormatter.JSON_CLASSNAME); for (Map.Entry entry : values.entrySet()) { pdxInstanceFactory.writeByteArray(entry.getKey(), entry.getValue().toArray()); } return pdxInstanceFactory.create(); } private Region getRegion(String table) { Region r = cache.getRegion(table); if (r == null) { try { if (isClient) { ClientRegionFactory crf = ((ClientCache) cache).createClientRegionFactory(ClientRegionShortcut.PROXY); r = crf.create(table); } else { RegionFactory rf = ((Cache) cache).createRegionFactory(RegionShortcut.PARTITION); r = rf.create(table); } } catch (RegionExistsException e) { // another thread created the region r = cache.getRegion(table); } } return r; } } ================================================ FILE: geode/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014-2016, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for Apache Geode (incubating). */ package com.yahoo.ycsb.db; ================================================ FILE: googlebigtable/README.md ================================================ # Google Bigtable Driver for YCSB This driver provides a YCSB workload binding for Google's hosted Bigtable, the inspiration for a number of key-value stores like HBase and Cassandra. The Bigtable Java client provides both Protobuf based GRPC and HBase client APIs. This binding implements the Protobuf API for testing the native client. To test Bigtable using the HBase API, see the `hbase10` binding. ## Quickstart ### 1. Setup a Bigtable Instance Login to the Google Cloud Console and follow the [Creating Instance](https://cloud.google.com/bigtable/docs/creating-instance) steps. Make a note of your instance ID and project ID. ### 2. Launch the Bigtable Shell From the Cloud Console, launch a shell and follow the [Quickstart](https://cloud.google.com/bigtable/docs/quickstart) up to step 4 where you launch the HBase shell. ### 3. Create a Table For best results, use the pre-splitting strategy recommended in [HBASE-4163](https://issues.apache.org/jira/browse/HBASE-4163): ``` hbase(main):001:0> n_splits = 200 # HBase recommends (10 * number of regionservers) hbase(main):002:0> create 'usertable', 'cf', {SPLITS => (1..n_splits).map {|i| "user#{1000+i*(9999-1000)/n_splits}"}} ``` Make a note of the column family, in this example it's `cf``. ### 4. Download JSON Credentials Follow these instructions for [Generating a JSON key](https://cloud.google.com/bigtable/docs/installing-hbase-shell#service-account) and save it to your host. ### 5. Load a Workload Switch to the root of the YCSB repo and choose the workload you want to run and `load` it first. With the CLI you must provide the column family and instance properties to load. ``` bin/ycsb load googlebigtable -p columnfamily=cf -p google.bigtable.project.id= -p google.bigtable.instance.id= -p google.bigtable.auth.json.keyfile= -P workloads/workloada ``` Make sure to replace the variables in the angle brackets above with the proper value from your instance. Additional configuration parameters are available below. The `load` step only executes inserts into the datastore. After loading data, run the same workload to mix reads with writes. ``` bin/ycsb run googlebigtable -p columnfamily=cf -p google.bigtable.project.id= -p google.bigtable.instance.id= -p google.bigtable.auth.json.keyfile= -P workloads/workloada ``` ## Configuration Options The following options can be configured using CLI (using the `-p` parameter) or hbase-site.xml (add the HBase config directory to YCSB's class path via CLI). Check the [Cloud Bigtable Client](https://github.com/manolama/cloud-bigtable-client) project for additional tuning parameters. * `columnfamily`: (Required) The Bigtable column family to target. * `google.bigtable.project.id`: (Required) The ID of a Bigtable project. * `google.bigtable.instance.id`: (Required) The name of a Bigtable instance. * `google.bigtable.auth.service.account.enable`: Whether or not to authenticate with a service account. The default is true. * `google.bigtable.auth.json.keyfile`: (Required) A service account key for authentication. * `debug`: If true, prints debug information to standard out. The default is false. * `clientbuffering`: Whether or not to use client side buffering and batching of write operations. This can significantly improve performance and defaults to true. ================================================ FILE: googlebigtable/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ googlebigtable-binding Google Cloud Bigtable Binding jar com.google.cloud.bigtable bigtable-hbase-1.0 ${googlebigtable.version} io.netty netty-tcnative-boringssl-static 1.1.33.Fork26 com.yahoo.ycsb core ${project.version} provided ================================================ FILE: googlebigtable/src/main/java/com/yahoo/ycsb/db/GoogleBigtableClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.io.IOException; import java.nio.charset.Charset; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.util.Bytes; import java.util.Set; import java.util.Vector; import java.util.concurrent.ExecutionException; import com.google.bigtable.repackaged.com.google.protobuf.ByteString; import com.google.bigtable.v2.Column; import com.google.bigtable.v2.Family; import com.google.bigtable.v2.MutateRowRequest; import com.google.bigtable.v2.Mutation; import com.google.bigtable.v2.ReadRowsRequest; import com.google.bigtable.v2.Row; import com.google.bigtable.v2.RowFilter; import com.google.bigtable.v2.RowRange; import com.google.bigtable.v2.RowSet; import com.google.bigtable.v2.Mutation.DeleteFromRow; import com.google.bigtable.v2.Mutation.SetCell; import com.google.bigtable.v2.RowFilter.Chain.Builder; import com.google.cloud.bigtable.config.BigtableOptions; import com.google.cloud.bigtable.grpc.BigtableDataClient; import com.google.cloud.bigtable.grpc.BigtableSession; import com.google.cloud.bigtable.grpc.BigtableTableName; import com.google.cloud.bigtable.grpc.async.AsyncExecutor; import com.google.cloud.bigtable.grpc.async.BulkMutation; import com.google.cloud.bigtable.hbase.BigtableOptionsFactory; import com.google.cloud.bigtable.util.ByteStringer; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; /** * Google Bigtable Proto client for YCSB framework. * * Bigtable offers two APIs. These include a native Protobuf GRPC API as well as * an HBase API wrapper for the GRPC API. This client implements the Protobuf * API to test the underlying calls wrapped up in the HBase API. To use the * HBase API, see the hbase10 client binding. */ public class GoogleBigtableClient extends com.yahoo.ycsb.DB { public static final Charset UTF8_CHARSET = Charset.forName("UTF8"); /** Property names for the CLI. */ private static final String ASYNC_MUTATOR_MAX_MEMORY = "mutatorMaxMemory"; private static final String ASYNC_MAX_INFLIGHT_RPCS = "mutatorMaxInflightRPCs"; private static final String CLIENT_SIDE_BUFFERING = "clientbuffering"; /** Tracks running thread counts so we know when to close the session. */ private static int threadCount = 0; /** This will load the hbase-site.xml config file and/or store CLI options. */ private static final Configuration CONFIG = HBaseConfiguration.create(); /** Print debug information to standard out. */ private boolean debug = false; /** Global Bigtable native API objects. */ private static BigtableOptions options; private static BigtableSession session; /** Thread loacal Bigtable native API objects. */ private BigtableDataClient client; private AsyncExecutor asyncExecutor; /** The column family use for the workload. */ private byte[] columnFamilyBytes; /** Cache for the last table name/ID to avoid byte conversions. */ private String lastTable = ""; private byte[] lastTableBytes; /** * If true, buffer mutations on the client. For measuring insert/update/delete * latencies, client side buffering should be disabled. */ private boolean clientSideBuffering = false; private BulkMutation bulkMutation; @Override public void init() throws DBException { Properties props = getProperties(); // Defaults the user can override if needed if (getProperties().containsKey(ASYNC_MUTATOR_MAX_MEMORY)) { CONFIG.set(BigtableOptionsFactory.BIGTABLE_BUFFERED_MUTATOR_MAX_MEMORY_KEY, getProperties().getProperty(ASYNC_MUTATOR_MAX_MEMORY)); } if (getProperties().containsKey(ASYNC_MAX_INFLIGHT_RPCS)) { CONFIG.set(BigtableOptionsFactory.BIGTABLE_BULK_MAX_ROW_KEY_COUNT, getProperties().getProperty(ASYNC_MAX_INFLIGHT_RPCS)); } // make it easy on ourselves by copying all CLI properties into the config object. final Iterator> it = props.entrySet().iterator(); while (it.hasNext()) { Entry entry = it.next(); CONFIG.set((String)entry.getKey(), (String)entry.getValue()); } clientSideBuffering = getProperties().getProperty(CLIENT_SIDE_BUFFERING, "false") .equals("true") ? true : false; System.err.println("Running Google Bigtable with Proto API" + (clientSideBuffering ? " and client side buffering." : ".")); synchronized (CONFIG) { ++threadCount; if (session == null) { try { options = BigtableOptionsFactory.fromConfiguration(CONFIG); session = new BigtableSession(options); // important to instantiate the first client here, otherwise the // other threads may receive an NPE from the options when they try // to read the cluster name. client = session.getDataClient(); } catch (IOException e) { throw new DBException("Error loading options from config: ", e); } } else { client = session.getDataClient(); } if (clientSideBuffering) { asyncExecutor = session.createAsyncExecutor(); } } if ((getProperties().getProperty("debug") != null) && (getProperties().getProperty("debug").compareTo("true") == 0)) { debug = true; } final String columnFamily = getProperties().getProperty("columnfamily"); if (columnFamily == null) { System.err.println("Error, must specify a columnfamily for Bigtable table"); throw new DBException("No columnfamily specified"); } columnFamilyBytes = Bytes.toBytes(columnFamily); } @Override public void cleanup() throws DBException { if (bulkMutation != null) { try { bulkMutation.flush(); } catch(RuntimeException e){ throw new DBException(e); } } if (asyncExecutor != null) { try { asyncExecutor.flush(); } catch (IOException e) { throw new DBException(e); } } synchronized (CONFIG) { --threadCount; if (threadCount <= 0) { try { session.close(); } catch (IOException e) { throw new DBException(e); } } } } @Override public Status read(String table, String key, Set fields, Map result) { if (debug) { System.out.println("Doing read from Bigtable columnfamily " + new String(columnFamilyBytes)); System.out.println("Doing read for key: " + key); } setTable(table); RowFilter filter = RowFilter.newBuilder() .setFamilyNameRegexFilterBytes(ByteStringer.wrap(columnFamilyBytes)) .build(); if (fields != null && fields.size() > 0) { Builder filterChain = RowFilter.Chain.newBuilder(); filterChain.addFilters(filter); filterChain.addFilters(RowFilter.newBuilder() .setCellsPerColumnLimitFilter(1) .build()); int count = 0; // usually "field#" so pre-alloc final StringBuilder regex = new StringBuilder(fields.size() * 6); for (final String field : fields) { if (count++ > 0) { regex.append("|"); } regex.append(field); } filterChain.addFilters(RowFilter.newBuilder() .setColumnQualifierRegexFilter( ByteStringer.wrap(regex.toString().getBytes()))).build(); filter = RowFilter.newBuilder().setChain(filterChain.build()).build(); } final ReadRowsRequest.Builder rrr = ReadRowsRequest.newBuilder() .setTableNameBytes(ByteStringer.wrap(lastTableBytes)) .setFilter(filter) .setRows(RowSet.newBuilder() .addRowKeys(ByteStringer.wrap(key.getBytes()))); List rows; try { rows = client.readRowsAsync(rrr.build()).get(); if (rows == null || rows.isEmpty()) { return Status.NOT_FOUND; } for (final Row row : rows) { for (final Family family : row.getFamiliesList()) { if (Arrays.equals(family.getNameBytes().toByteArray(), columnFamilyBytes)) { for (final Column column : family.getColumnsList()) { // we should only have a single cell per column result.put(column.getQualifier().toString(UTF8_CHARSET), new ByteArrayByteIterator(column.getCells(0).getValue().toByteArray())); if (debug) { System.out.println( "Result for field: " + column.getQualifier().toString(UTF8_CHARSET) + " is: " + column.getCells(0).getValue().toString(UTF8_CHARSET)); } } } } } return Status.OK; } catch (InterruptedException e) { System.err.println("Interrupted during get: " + e); Thread.currentThread().interrupt(); return Status.ERROR; } catch (ExecutionException e) { System.err.println("Exception during get: " + e); return Status.ERROR; } } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { setTable(table); RowFilter filter = RowFilter.newBuilder() .setFamilyNameRegexFilterBytes(ByteStringer.wrap(columnFamilyBytes)) .build(); if (fields != null && fields.size() > 0) { Builder filterChain = RowFilter.Chain.newBuilder(); filterChain.addFilters(filter); filterChain.addFilters(RowFilter.newBuilder() .setCellsPerColumnLimitFilter(1) .build()); int count = 0; // usually "field#" so pre-alloc final StringBuilder regex = new StringBuilder(fields.size() * 6); for (final String field : fields) { if (count++ > 0) { regex.append("|"); } regex.append(field); } filterChain.addFilters(RowFilter.newBuilder() .setColumnQualifierRegexFilter( ByteStringer.wrap(regex.toString().getBytes()))).build(); filter = RowFilter.newBuilder().setChain(filterChain.build()).build(); } final RowRange range = RowRange.newBuilder() .setStartKeyClosed(ByteStringer.wrap(startkey.getBytes())) .build(); final RowSet rowSet = RowSet.newBuilder() .addRowRanges(range) .build(); final ReadRowsRequest.Builder rrr = ReadRowsRequest.newBuilder() .setTableNameBytes(ByteStringer.wrap(lastTableBytes)) .setFilter(filter) .setRows(rowSet); List rows; try { rows = client.readRowsAsync(rrr.build()).get(); if (rows == null || rows.isEmpty()) { return Status.NOT_FOUND; } int numResults = 0; for (final Row row : rows) { final HashMap rowResult = new HashMap(fields != null ? fields.size() : 10); for (final Family family : row.getFamiliesList()) { if (Arrays.equals(family.getNameBytes().toByteArray(), columnFamilyBytes)) { for (final Column column : family.getColumnsList()) { // we should only have a single cell per column rowResult.put(column.getQualifier().toString(UTF8_CHARSET), new ByteArrayByteIterator(column.getCells(0).getValue().toByteArray())); if (debug) { System.out.println( "Result for field: " + column.getQualifier().toString(UTF8_CHARSET) + " is: " + column.getCells(0).getValue().toString(UTF8_CHARSET)); } } } } result.add(rowResult); numResults++; if (numResults >= recordcount) {// if hit recordcount, bail out break; } } return Status.OK; } catch (InterruptedException e) { System.err.println("Interrupted during scan: " + e); Thread.currentThread().interrupt(); return Status.ERROR; } catch (ExecutionException e) { System.err.println("Exception during scan: " + e); return Status.ERROR; } } @Override public Status update(String table, String key, Map values) { if (debug) { System.out.println("Setting up put for key: " + key); } setTable(table); final MutateRowRequest.Builder rowMutation = MutateRowRequest.newBuilder(); rowMutation.setRowKey(ByteString.copyFromUtf8(key)); rowMutation.setTableNameBytes(ByteStringer.wrap(lastTableBytes)); for (final Entry entry : values.entrySet()) { final Mutation.Builder mutationBuilder = rowMutation.addMutationsBuilder(); final SetCell.Builder setCellBuilder = mutationBuilder.getSetCellBuilder(); setCellBuilder.setFamilyNameBytes(ByteStringer.wrap(columnFamilyBytes)); setCellBuilder.setColumnQualifier(ByteStringer.wrap(entry.getKey().getBytes())); setCellBuilder.setValue(ByteStringer.wrap(entry.getValue().toArray())); // Bigtable uses a 1ms granularity setCellBuilder.setTimestampMicros(System.currentTimeMillis() * 1000); } try { if (clientSideBuffering) { bulkMutation.add(rowMutation.build()); } else { client.mutateRow(rowMutation.build()); } return Status.OK; } catch (RuntimeException e) { System.err.println("Failed to insert key: " + key + " " + e.getMessage()); return Status.ERROR; } } @Override public Status insert(String table, String key, Map values) { return update(table, key, values); } @Override public Status delete(String table, String key) { if (debug) { System.out.println("Doing delete for key: " + key); } setTable(table); final MutateRowRequest.Builder rowMutation = MutateRowRequest.newBuilder() .setRowKey(ByteString.copyFromUtf8(key)) .setTableNameBytes(ByteStringer.wrap(lastTableBytes)); rowMutation.addMutationsBuilder().setDeleteFromRow( DeleteFromRow.getDefaultInstance()); try { if (clientSideBuffering) { bulkMutation.add(rowMutation.build()); } else { client.mutateRow(rowMutation.build()); } return Status.OK; } catch (RuntimeException e) { System.err.println("Failed to delete key: " + key + " " + e.getMessage()); return Status.ERROR; } } /** * Little helper to set the table byte array. If it's different than the last * table we reset the byte array. Otherwise we just use the existing array. * @param table The table we're operating against */ private void setTable(final String table) { if (!lastTable.equals(table)) { lastTable = table; BigtableTableName tableName = options .getInstanceName() .toTableName(table); lastTableBytes = tableName .toString() .getBytes(); synchronized(this) { if (bulkMutation != null) { bulkMutation.flush(); } bulkMutation = session.createBulkMutation(tableName, asyncExecutor); } } } } ================================================ FILE: googlebigtable/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Google's * Bigtable. */ package com.yahoo.ycsb.db; ================================================ FILE: googledatastore/README.md ================================================ # Google Cloud Datastore Binding https://cloud.google.com/datastore/docs/concepts/overview?hl=en Please refer [here] (https://cloud.google.com/datastore/docs/apis/overview) for more information on Google Cloud Datastore API. ## Configure YCSB_HOME - YCSB home directory DATASTORE_HOME - Google Cloud Datastore YCSB client package files Please refer to https://github.com/brianfrankcooper/YCSB/wiki/Using-the-Database-Libraries for more information on setup. # Benchmark $YCSB_HOME/bin/ycsb load googledatastore -P workloads/workloada -P googledatastore.properties $YCSB_HOME/bin/ycsb run googledatastore -P workloads/workloada -P googledatastore.properties # Properties $DATASTORE_HOME/conf/googledatastore.properties # Details A. Configuration and setup: See this link for instructions about setting up Google Cloud Datastore and authentication: https://cloud.google.com/datastore/docs/activate#accessing_the_datastore_api_from_another_platform After you setup your environment, you will have 3 pieces of information ready: - datasetId, - service account email, and - a private key file in P12 format. These will be configured via corresponding properties in the googledatastore.properties file. B. EntityGroupingMode In Google Datastore, Entity Group is the unit in which the user can perform strongly consistent query on multiple items; Meanwhile, Entity group also has certain limitations in performance, especially with write QPS. We support two modes here: 1. [default] One entity per group (ONE_ENTITY_PER_GROUP) In this mode, every entity is a "root" entity and sits in one group, and every entity group has only one entity. Write QPS is high in this mode (and there is no documented limitation on this). But query across multiple entities are eventually consistent. When this mode is set, every entity is created with no ancestor key (meaning the entity itself is the "root" entity). 2. Multiple entities per group (MULTI_ENTITY_PER_GROUP) In this mode, all entities in one benchmark run are placed under one ancestor (root) node therefore inside one entity group. Query/scan performed on these entities will be strongly consistent but write QPS will be subject to documented limitation (current is at 1 QPS). Because of the write QPS limit, it's highly recommended that you rate limit your benchmark's test rate to avoid excessive errors. The goal of this MULTI_ENTITY_PER_GROUP mode is to allow user to benchmark and understand performance characteristics of a single entity group of the Google Datastore. While in this mode, one can optionally specify a root key name. If not specified, a default name will be used. ================================================ FILE: googledatastore/conf/googledatastore.properties ================================================ # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Sample property file for Google Cloud Datastore DB client ## Mandatory parameters # # Your credentials to Google datastore. See README.md for details. # # googledatastore.datasetId= # googledatastore.privateKeyFile= # googledatastore.serviceAccountEmail= # Google Cloud Datastore's read and update APIs do not support # reading or updating a select subset of properties for an entity. # (as of version v1beta3) # Therefore, it's recommended that you set writeallfields and readallfields # to true to get stable and comparable performance numbers. writeallfields = true readallfields = true ## Optional parameters # # Decides the consistency level of read requests. Acceptable values are: # EVENTUAL, STRONG (default is STRONG) # # googledatastore.readConsistency=STRONG # Decides how we group entities into entity groups. # (See the details section in README.md for documentation) # # googledatastore.entityGroupingMode=ONE_ENTITY_PER_GROUP # If you set the googledatastore.entityGroupingMode property to # MULTI_ENTITY_PER_GROUP, you can optionally specify the name of the root entity # # googledatastore.rootEntityName="YCSB_ROOT_ENTITY" # Strongly recommended to set to uniform. # requestdistribution = uniform # Enable/disable debug message, default is false. # googledatastore.debug = false # Skip indexes, default is true. # googledatastore.skipIndex = true ================================================ FILE: googledatastore/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent googledatastore-binding Google Cloud Datastore Binding https://github.com/GoogleCloudPlatform/google-cloud-datastore com.google.cloud.datastore datastore-v1-proto-client 1.1.0 log4j log4j 1.2.17 com.yahoo.ycsb core ${project.version} provided ================================================ FILE: googledatastore/src/main/java/com/yahoo/ycsb/db/GoogleDatastoreClient.java ================================================ /* * Copyright 2015 YCSB contributors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.datastore.v1.*; import com.google.datastore.v1.CommitRequest.Mode; import com.google.datastore.v1.ReadOptions.ReadConsistency; import com.google.datastore.v1.client.Datastore; import com.google.datastore.v1.client.DatastoreException; import com.google.datastore.v1.client.DatastoreFactory; import com.google.datastore.v1.client.DatastoreHelper; import com.google.datastore.v1.client.DatastoreOptions; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.apache.log4j.Level; import org.apache.log4j.Logger; import java.io.IOException; import java.security.GeneralSecurityException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.Vector; import javax.annotation.Nullable; /** * Google Cloud Datastore Client for YCSB. */ public class GoogleDatastoreClient extends DB { /** * Defines a MutationType used in this class. */ private enum MutationType { UPSERT, UPDATE, DELETE } /** * Defines a EntityGroupingMode enum used in this class. */ private enum EntityGroupingMode { ONE_ENTITY_PER_GROUP, MULTI_ENTITY_PER_GROUP } private static Logger logger = Logger.getLogger(GoogleDatastoreClient.class); // Read consistency defaults to "STRONG" per YCSB guidance. // User can override this via configure. private ReadConsistency readConsistency = ReadConsistency.STRONG; private EntityGroupingMode entityGroupingMode = EntityGroupingMode.ONE_ENTITY_PER_GROUP; private String rootEntityName; private Datastore datastore = null; private static boolean skipIndex = true; /** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. */ @Override public void init() throws DBException { String debug = getProperties().getProperty("googledatastore.debug", null); if (null != debug && "true".equalsIgnoreCase(debug)) { logger.setLevel(Level.DEBUG); } String skipIndexString = getProperties().getProperty( "googledatastore.skipIndex", null); if (null != skipIndexString && "false".equalsIgnoreCase(skipIndexString)) { skipIndex = false; } // We need the following 3 essential properties to initialize datastore: // // - DatasetId, // - Path to private key file, // - Service account email address. String datasetId = getProperties().getProperty( "googledatastore.datasetId", null); if (datasetId == null) { throw new DBException( "Required property \"datasetId\" missing."); } String privateKeyFile = getProperties().getProperty( "googledatastore.privateKeyFile", null); String serviceAccountEmail = getProperties().getProperty( "googledatastore.serviceAccountEmail", null); // Below are properties related to benchmarking. String readConsistencyConfig = getProperties().getProperty( "googledatastore.readConsistency", null); if (readConsistencyConfig != null) { try { this.readConsistency = ReadConsistency.valueOf( readConsistencyConfig.trim().toUpperCase()); } catch (IllegalArgumentException e) { throw new DBException("Invalid read consistency specified: " + readConsistencyConfig + ". Expecting STRONG or EVENTUAL."); } } // // Entity Grouping Mode (googledatastore.entitygroupingmode), see // documentation in conf/googledatastore.properties. // String entityGroupingConfig = getProperties().getProperty( "googledatastore.entityGroupingMode", null); if (entityGroupingConfig != null) { try { this.entityGroupingMode = EntityGroupingMode.valueOf( entityGroupingConfig.trim().toUpperCase()); } catch (IllegalArgumentException e) { throw new DBException("Invalid entity grouping mode specified: " + entityGroupingConfig + ". Expecting ONE_ENTITY_PER_GROUP or " + "MULTI_ENTITY_PER_GROUP."); } } this.rootEntityName = getProperties().getProperty( "googledatastore.rootEntityName", "YCSB_ROOT_ENTITY"); try { // Setup the connection to Google Cloud Datastore with the credentials // obtained from the configure. DatastoreOptions.Builder options = new DatastoreOptions.Builder(); Credential credential = GoogleCredential.getApplicationDefault(); if (serviceAccountEmail != null && privateKeyFile != null) { credential = DatastoreHelper.getServiceAccountCredential( serviceAccountEmail, privateKeyFile); logger.info("Using JWT Service Account credential."); logger.info("DatasetID: " + datasetId + ", Service Account Email: " + serviceAccountEmail + ", Private Key File Path: " + privateKeyFile); } else { logger.info("Using default gcloud credential."); logger.info("DatasetID: " + datasetId + ", Service Account Email: " + ((GoogleCredential) credential).getServiceAccountId()); } datastore = DatastoreFactory.get().create( options.credential(credential).projectId(datasetId).build()); } catch (GeneralSecurityException exception) { throw new DBException("Security error connecting to the datastore: " + exception.getMessage(), exception); } catch (IOException exception) { throw new DBException("I/O error connecting to the datastore: " + exception.getMessage(), exception); } logger.info("Datastore client instance created: " + datastore.toString()); } @Override public Status read(String table, String key, Set fields, Map result) { LookupRequest.Builder lookupRequest = LookupRequest.newBuilder(); lookupRequest.addKeys(buildPrimaryKey(table, key)); lookupRequest.getReadOptionsBuilder().setReadConsistency( this.readConsistency); // Note above, datastore lookupRequest always reads the entire entity, it // does not support reading a subset of "fields" (properties) of an entity. logger.debug("Built lookup request as: " + lookupRequest.toString()); LookupResponse response = null; try { response = datastore.lookup(lookupRequest.build()); } catch (DatastoreException exception) { logger.error( String.format("Datastore Exception when reading (%s): %s %s", exception.getMessage(), exception.getMethodName(), exception.getCode())); // DatastoreException.getCode() returns an HTTP response code which we // will bubble up to the user as part of the YCSB Status "name". return new Status("ERROR-" + exception.getCode(), exception.getMessage()); } if (response.getFoundCount() == 0) { return new Status("ERROR-404", "Not Found, key is: " + key); } else if (response.getFoundCount() > 1) { // We only asked to lookup for one key, shouldn't have got more than one // entity back. Unexpected State. return Status.UNEXPECTED_STATE; } Entity entity = response.getFound(0).getEntity(); logger.debug("Read entity: " + entity.toString()); Map properties = entity.getProperties(); Set propertiesToReturn = (fields == null ? properties.keySet() : fields); for (String name : propertiesToReturn) { if (properties.containsKey(name)) { result.put(name, new StringByteIterator(properties.get(name) .getStringValue())); } } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // TODO: Implement Scan as query on primary key. return Status.NOT_IMPLEMENTED; } @Override public Status update(String table, String key, Map values) { return doSingleItemMutation(table, key, values, MutationType.UPDATE); } @Override public Status insert(String table, String key, Map values) { // Use Upsert to allow overwrite of existing key instead of failing the // load (or run) just because the DB already has the key. // This is the same behavior as what other DB does here (such as // the DynamoDB client). return doSingleItemMutation(table, key, values, MutationType.UPSERT); } @Override public Status delete(String table, String key) { return doSingleItemMutation(table, key, null, MutationType.DELETE); } private Key.Builder buildPrimaryKey(String table, String key) { Key.Builder result = Key.newBuilder(); if (this.entityGroupingMode == EntityGroupingMode.MULTI_ENTITY_PER_GROUP) { // All entities are in side the same group when we are in this mode. result.addPath(Key.PathElement.newBuilder().setKind(table). setName(rootEntityName)); } return result.addPath(Key.PathElement.newBuilder().setKind(table) .setName(key)); } private Status doSingleItemMutation(String table, String key, @Nullable Map values, MutationType mutationType) { // First build the key. Key.Builder datastoreKey = buildPrimaryKey(table, key); // Build a commit request in non-transactional mode. // Single item mutation to google datastore // is always atomic and strongly consistent. Transaction is only necessary // for multi-item mutation, or Read-modify-write operation. CommitRequest.Builder commitRequest = CommitRequest.newBuilder(); commitRequest.setMode(Mode.NON_TRANSACTIONAL); if (mutationType == MutationType.DELETE) { commitRequest.addMutationsBuilder().setDelete(datastoreKey); } else { // If this is not for delete, build the entity. Entity.Builder entityBuilder = Entity.newBuilder(); entityBuilder.setKey(datastoreKey); for (Entry val : values.entrySet()) { entityBuilder.getMutableProperties() .put(val.getKey(), Value.newBuilder() .setStringValue(val.getValue().toString()) .setExcludeFromIndexes(skipIndex).build()); } Entity entity = entityBuilder.build(); logger.debug("entity built as: " + entity.toString()); if (mutationType == MutationType.UPSERT) { commitRequest.addMutationsBuilder().setUpsert(entity); } else if (mutationType == MutationType.UPDATE){ commitRequest.addMutationsBuilder().setUpdate(entity); } else { throw new RuntimeException("Impossible MutationType, code bug."); } } try { datastore.commit(commitRequest.build()); logger.debug("successfully committed."); } catch (DatastoreException exception) { // Catch all Datastore rpc errors. // Log the exception, the name of the method called and the error code. logger.error( String.format("Datastore Exception when committing (%s): %s %s", exception.getMessage(), exception.getMethodName(), exception.getCode())); // DatastoreException.getCode() returns an HTTP response code which we // will bubble up to the user as part of the YCSB Status "name". return new Status("ERROR-" + exception.getCode(), exception.getMessage()); } return Status.OK; } } ================================================ FILE: googledatastore/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for Google Cloud Datastore. */ package com.yahoo.ycsb.db; ================================================ FILE: googledatastore/src/main/resources/log4j.properties ================================================ # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. #define the console appender log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender # now define the layout for the appender log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n # now map our console appender as a root logger, means all log messages will go # to this appender log4j.rootLogger = INFO, consoleAppender ================================================ FILE: hbase098/README.md ================================================ # HBase (0.98.x) Driver for YCSB This driver is a binding for the YCSB facilities to operate against a HBase 0.98.x Server cluster. To run against an HBase >= 1.0 cluster, use the `hbase10` binding. ## Quickstart ### 1. Start a HBase Server You need to start a single node or a cluster to point the client at. Please see [Apache HBase Reference Guide](http://hbase.apache.org/book.html) for more details and instructions. ### 2. Set up YCSB You need to clone the repository and compile everything. ``` git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ``` ### 3. Create a HBase table for testing For best results, use the pre-splitting strategy recommended in [HBASE-4163](https://issues.apache.org/jira/browse/HBASE-4163): ``` hbase(main):001:0> n_splits = 200 # HBase recommends (10 * number of regionservers) hbase(main):002:0> create 'usertable', 'family', {SPLITS => (1..n_splits).map {|i| "user#{1000+i*(9999-1000)/n_splits}"}} ``` *Failing to do so will cause all writes to initially target a single region server*. ### 4. Run the Workload Before you can actually run the workload, you need to "load" the data first. You should specify a HBase config directory(or any other directory containing your hbase-site.xml) and a table name and a column family(-cp is used to set java classpath and -p is used to set various properties). ``` bin/ycsb load hbase -P workloads/workloada -cp /HBASE-HOME-DIR/conf -p table=usertable -p columnfamily=family ``` Then, you can run the workload: ``` bin/ycsb run hbase -P workloads/workloada -cp /HBASE-HOME-DIR/conf -p table=usertable -p columnfamily=family ``` Please see the general instructions in the `doc` folder if you are not sure how it all works. You can apply additional properties (as seen in the next section) like this: ``` bin/ycsb run hbase -P workloads/workloada -cp /HBASE-HOME-DIR/conf -p table=usertable -p columnfamily=family -p clientbuffering=true ``` ## Configuration Options Following options can be configurable using `-p`. * `columnfamily`: The HBase column family to target. * `debug` : If true, debugging logs are activated. The default is false. * `hbase.usepagefilter` : If true, HBase [PageFilter](https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/PageFilter.html)s are used to limit the number of records consumed in a scan operation. The default is true. * `principal`: If testing need to be done against a secure HBase cluster using Kerberos Keytab, this property can be used to pass the principal in the keytab file. * `keytab`: The Kerberos keytab file name and location can be passed through this property. * `writebuffersize`: The maximum amount, in bytes, of data to buffer on the client side before a flush is forced. The default is 12MB. Additional HBase settings should be provided in the `hbase-site.xml` file located in your `/HBASE-HOME-DIR/conf` directory. Typically this will be `/etc/hbase/conf`. ================================================ FILE: hbase098/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ hbase098-binding HBase 0.98.x DB Binding org.apache.hbase hbase-client ${hbase098.version} jdk.tools jdk.tools com.yahoo.ycsb core ${project.version} provided ================================================ FILE: hbase098/src/main/java/com/yahoo/ycsb/db/HBaseClient.java ================================================ /** * Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.measurements.Measurements; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; /** * HBase client for YCSB framework. */ public class HBaseClient extends com.yahoo.ycsb.DB { private static final Configuration CONFIG = HBaseConfiguration.create(); private static final AtomicInteger THREAD_COUNT = new AtomicInteger(0); private boolean debug = false; private String tableName = ""; private static HConnection hConn = null; private HTableInterface hTable = null; private String columnFamily = ""; private byte[] columnFamilyBytes; private boolean clientSideBuffering = false; private long writeBufferSize = 1024 * 1024 * 12; /** * Whether or not a page filter should be used to limit scan length. */ private boolean usePageFilter = true; private static final Object TABLE_LOCK = new Object(); /** * Initialize any state for this DB. * Called once per DB instance; there is one DB instance per client thread. */ public void init() throws DBException { if ((getProperties().getProperty("debug") != null) && (getProperties().getProperty("debug").compareTo("true") == 0)) { debug = true; } if (getProperties().containsKey("clientbuffering")) { clientSideBuffering = Boolean.parseBoolean(getProperties().getProperty("clientbuffering")); } if (getProperties().containsKey("writebuffersize")) { writeBufferSize = Long.parseLong(getProperties().getProperty("writebuffersize")); } if ("false".equals(getProperties().getProperty("hbase.usepagefilter", "true"))) { usePageFilter = false; } if ("kerberos".equalsIgnoreCase(CONFIG.get("hbase.security.authentication"))) { CONFIG.set("hadoop.security.authentication", "Kerberos"); UserGroupInformation.setConfiguration(CONFIG); } if ((getProperties().getProperty("principal") != null) && (getProperties().getProperty("keytab") != null)) { try { UserGroupInformation.loginUserFromKeytab(getProperties().getProperty("principal"), getProperties().getProperty("keytab")); } catch (IOException e) { System.err.println("Keytab file is not readable or not found"); throw new DBException(e); } } try { THREAD_COUNT.getAndIncrement(); synchronized (THREAD_COUNT) { if (hConn == null) { hConn = HConnectionManager.createConnection(CONFIG); } } } catch (IOException e) { System.err.println("Connection to HBase was not successful"); throw new DBException(e); } columnFamily = getProperties().getProperty("columnfamily"); if (columnFamily == null) { System.err.println("Error, must specify a columnfamily for HBase tableName"); throw new DBException("No columnfamily specified"); } columnFamilyBytes = Bytes.toBytes(columnFamily); // Terminate right now if tableName does not exist, since the client // will not propagate this error upstream once the workload // starts. String table = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); try { HTableInterface ht = hConn.getTable(table); ht.getTableDescriptor(); } catch (IOException e) { throw new DBException(e); } } /** * Cleanup any state for this DB. * Called once per DB instance; there is one DB instance per client thread. */ public void cleanup() throws DBException { // Get the measurements instance as this is the only client that should // count clean up time like an update since autoflush is off. Measurements measurements = Measurements.getMeasurements(); try { long st = System.nanoTime(); if (hTable != null) { hTable.flushCommits(); } synchronized (THREAD_COUNT) { int threadCount = THREAD_COUNT.decrementAndGet(); if (threadCount <= 0 && hConn != null) { hConn.close(); } } long en = System.nanoTime(); measurements.measure("UPDATE", (int) ((en - st) / 1000)); } catch (IOException e) { throw new DBException(e); } } private void getHTable(String table) throws IOException { synchronized (TABLE_LOCK) { hTable = hConn.getTable(table); //2 suggestions from http://ryantwopointoh.blogspot.com/2009/01/performance-of-hbase-importing.html hTable.setAutoFlush(!clientSideBuffering, true); hTable.setWriteBufferSize(writeBufferSize); //return hTable; } } /** * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the tableName * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ public Status read(String table, String key, Set fields, Map result) { //if this is a "new" tableName, init HTable object. Else, use existing one if (!this.tableName.equals(table)) { hTable = null; try { getHTable(table); this.tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase tableName: " + e); return Status.ERROR; } } Result r; try { if (debug) { System.out.println("Doing read from HBase columnfamily " + columnFamily); System.out.println("Doing read for key: " + key); } Get g = new Get(Bytes.toBytes(key)); if (fields == null) { g.addFamily(columnFamilyBytes); } else { for (String field : fields) { g.addColumn(columnFamilyBytes, Bytes.toBytes(field)); } } r = hTable.get(g); } catch (IOException e) { System.err.println("Error doing get: " + e); return Status.ERROR; } catch (ConcurrentModificationException e) { //do nothing for now...need to understand HBase concurrency model better return Status.ERROR; } for (KeyValue kv : r.raw()) { result.put( Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue())); if (debug) { System.out.println("Result for field: " + Bytes.toString(kv.getQualifier()) + " is: " + Bytes.toString(kv.getValue())); } } return Status.OK; } /** * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored * in a HashMap. * * @param table The name of the tableName * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return Zero on success, a non-zero error code on error */ public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { //if this is a "new" tableName, init HTable object. Else, use existing one if (!this.tableName.equals(table)) { hTable = null; try { getHTable(table); this.tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase tableName: " + e); return Status.ERROR; } } Scan s = new Scan(Bytes.toBytes(startkey)); //HBase has no record limit. Here, assume recordcount is small enough to bring back in one call. //We get back recordcount records s.setCaching(recordcount); if (this.usePageFilter) { s.setFilter(new PageFilter(recordcount)); } //add specified fields or else all fields if (fields == null) { s.addFamily(columnFamilyBytes); } else { for (String field : fields) { s.addColumn(columnFamilyBytes, Bytes.toBytes(field)); } } //get results try (ResultScanner scanner = hTable.getScanner(s)) { int numResults = 0; for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { //get row key String key = Bytes.toString(rr.getRow()); if (debug) { System.out.println("Got scan result for key: " + key); } HashMap rowResult = new HashMap<>(); for (KeyValue kv : rr.raw()) { rowResult.put( Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue())); } //add rowResult to result vector result.add(rowResult); numResults++; // PageFilter does not guarantee that the number of results is <= pageSize, so this // break is required. //if hit recordcount, bail out if (numResults >= recordcount) { break; } } //done with row } catch (IOException e) { if (debug) { System.out.println("Error in getting/parsing scan result: " + e); } return Status.ERROR; } return Status.OK; } /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key, overwriting any existing values with the same field name. * * @param table The name of the tableName * @param key The record key of the record to write * @param values A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error */ public Status update(String table, String key, Map values) { //if this is a "new" tableName, init HTable object. Else, use existing one if (!this.tableName.equals(table)) { hTable = null; try { getHTable(table); this.tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase tableName: " + e); return Status.ERROR; } } if (debug) { System.out.println("Setting up put for key: " + key); } Put p = new Put(Bytes.toBytes(key)); for (Map.Entry entry : values.entrySet()) { byte[] value = entry.getValue().toArray(); if (debug) { System.out.println("Adding field/value " + entry.getKey() + "/" + Bytes.toStringBinary(value) + " to put request"); } p.add(columnFamilyBytes, Bytes.toBytes(entry.getKey()), value); } try { hTable.put(p); } catch (IOException e) { if (debug) { System.err.println("Error doing put: " + e); } return Status.ERROR; } catch (ConcurrentModificationException e) { //do nothing for now...hope this is rare return Status.ERROR; } return Status.OK; } /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key. * * @param table The name of the tableName * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error */ public Status insert(String table, String key, Map values) { return update(table, key, values); } /** * Delete a record from the database. * * @param table The name of the tableName * @param key The record key of the record to delete. * @return Zero on success, a non-zero error code on error */ public Status delete(String table, String key) { //if this is a "new" tableName, init HTable object. Else, use existing one if (!this.tableName.equals(table)) { hTable = null; try { getHTable(table); this.tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase tableName: " + e); return Status.ERROR; } } if (debug) { System.out.println("Doing delete for key: " + key); } Delete d = new Delete(Bytes.toBytes(key)); try { hTable.delete(d); } catch (IOException e) { if (debug) { System.err.println("Error doing delete: " + e); } return Status.ERROR; } return Status.OK; } public static void main(String[] args) { if (args.length != 3) { System.out.println("Please specify a threadcount, columnfamily and operation count"); System.exit(0); } final int keyspace = 10000; //120000000; final int threadcount = Integer.parseInt(args[0]); final String columnfamily = args[1]; final int opcount = Integer.parseInt(args[2]) / threadcount; Vector allthreads = new Vector<>(); for (int i = 0; i < threadcount; i++) { Thread t = new Thread() { public void run() { try { Random random = new Random(); HBaseClient cli = new HBaseClient(); Properties props = new Properties(); props.setProperty("columnfamily", columnfamily); props.setProperty("debug", "true"); cli.setProperties(props); cli.init(); long accum = 0; for (int i = 0; i < opcount; i++) { int keynum = random.nextInt(keyspace); String key = "user" + keynum; long st = System.currentTimeMillis(); Status result; Vector> scanResults = new Vector<>(); Set scanFields = new HashSet(); result = cli.scan("table1", "user2", 20, null, scanResults); long en = System.currentTimeMillis(); accum += (en - st); if (!result.equals(Status.OK)) { System.out.println("Error " + result + " for " + key); } if (i % 10 == 0) { System.out.println(i + " operations, average latency: " + (((double) accum) / ((double) i))); } } } catch (Exception e) { e.printStackTrace(); } } }; allthreads.add(t); } long st = System.currentTimeMillis(); for (Thread t : allthreads) { t.start(); } for (Thread t : allthreads) { try { t.join(); } catch (InterruptedException ignored) { System.err.println("interrupted"); Thread.currentThread().interrupt(); } } long en = System.currentTimeMillis(); System.out.println("Throughput: " + ((1000.0) * (((double) (opcount * threadcount)) / ((double) (en - st)))) + " ops/sec"); } } ================================================ FILE: hbase098/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2017, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. See accompanying LICENSE file. */ /** * The YCSB binding for HBase * 0.98.X. */ package com.yahoo.ycsb.db; ================================================ FILE: hbase10/README.md ================================================ # HBase (1.0.x) Driver for YCSB This driver is a binding for the YCSB facilities to operate against a HBase 1.0.x Server cluster or Google's hosted Bigtable. To run against an HBase 0.98.x cluster, use the `hbase098` binding. See `hbase098/README.md` for a quickstart to setup HBase for load testing and common configuration details. ## Configuration Options In addition to those options available for the `hbase098` binding, the following options are available for the `hbase10` binding: * `durability`: Whether or not writes should be appended to the WAL. Bypassing the WAL can improve throughput but data cannot be recovered in the event of a crash. The default is true. ## Bigtable Google's Bigtable service provides an implementation of the HBase API for migrating existing applications. Users can perform load tests against Bigtable using this binding. ### 1. Setup a Bigtable Cluster Login to the Google Cloud Console and follow the [Creating Cluster](https://cloud.google.com/bigtable/docs/creating-cluster) steps. Make a note of your cluster name, zone and project ID. ### 2. Launch the Bigtable Shell From the Cloud Console, launch a shell and follow the [Quickstart](https://cloud.google.com/bigtable/docs/quickstart) up to step 4 where you launch the HBase shell. ### 3. Create a Table For best results, use the pre-splitting strategy recommended in [HBASE-4163](https://issues.apache.org/jira/browse/HBASE-4163): ``` hbase(main):001:0> n_splits = 200 # HBase recommends (10 * number of regionservers) hbase(main):002:0> create 'usertable', 'cf', {SPLITS => (1..n_splits).map {|i| "user#{1000+i*(9999-1000)/n_splits}"}} ``` Make a note of the column family, in this example it's `cf``. ### 4. Fetch the Proper ALPN Boot Jar The Bigtable protocol uses HTTP/2 which requires an ALPN protocol negotiation implementation. On JVM instantiation the implementation must be loaded before attempting to connect to the cluster. If you're using Java 7 or 8, use this [Jetty Version Table](http://www.eclipse.org/jetty/documentation/current/alpn-chapter.html#alpn-versions) to determine the version appropriate for your JVM. (ALPN is included in JDK 9+). Download the proper jar from [Maven](http://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22org.mortbay.jetty.alpn%22%20AND%20a%3A%22alpn-boot%22) somewhere on your system. ### 5. Download the Bigtable Client Jar Download one of the `bigtable-hbase-1.#` jars from [Maven](http://search.maven.org/#search%7Cga%7C1%7Ccom.google.cloud.bigtable) to your host. ### 6. Download JSON Credentials Follow these instructions for [Generating a JSON key](https://cloud.google.com/bigtable/docs/installing-hbase-shell#service-account) and save it to your host. ### 7. Create or Edit hbase-site.xml If you have an existing HBase configuration directory with an `hbase-site.xml` file, edit the file as per below. If not, create a directory called `conf` under the `hbase10` directory. Create a file in the conf directory named `hbase-site.xml`. Provide the following settings in the XML file, making sure to replace the bracketed examples with the proper values from your Cloud console. ``` hbase.client.connection.impl com.google.cloud.bigtable.hbase1_0.BigtableConnection google.bigtable.cluster.name [YOUR-CLUSTER-ID] google.bigtable.project.id [YOUR-PROJECT-ID] google.bigtable.zone.name [YOUR-ZONE-NAME] google.bigtable.auth.service.account.enable true google.bigtable.auth.json.keyfile [PATH-TO-YOUR-KEY-FILE] ``` If you wish to try other API implementations (1.1.x or 1.2.x) change the `hbase.client.connection.impl` appropriately to match the JAR you downloaded. If you have an existing HBase config directory, make sure to add it to the class path via `-cp :`. ### 8. Execute a Workload Switch to the root of the YCSB repo and choose the workload you want to run and `load` it first. With the CLI you must provide the column family, cluster properties and the ALPN jar to load. ``` bin/ycsb load hbase10 -p columnfamily=cf -cp -jvm-args='-Xbootclasspath/p:' -P workloads/workloada ``` The `load` step only executes inserts into the datastore. After loading data, run the same workload to mix reads with writes. ``` bin/ycsb run hbase10 -p columnfamily=cf -jvm-args='-Xbootclasspath/p:' -P workloads/workloada ``` ================================================ FILE: hbase10/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ hbase10-binding HBase 1.0 DB Binding true org.apache.hbase hbase-client ${hbase10.version} jdk.tools jdk.tools com.yahoo.ycsb core ${project.version} provided junit junit 4.12 test org.apache.hbase hbase-testing-util ${hbase10.version} test jdk.tools jdk.tools ================================================ FILE: hbase10/src/main/java/com/yahoo/ycsb/db/HBaseClient10.java ================================================ /** * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.measurements.Measurements; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.BufferedMutatorParams; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.util.Bytes; import java.io.IOException; import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; /** * HBase 1.0 client for YCSB framework. * * A modified version of HBaseClient (which targets HBase v0.9) utilizing the * HBase 1.0.0 API. * * This client also adds toggleable client-side buffering and configurable write * durability. */ public class HBaseClient10 extends com.yahoo.ycsb.DB { private static final AtomicInteger THREAD_COUNT = new AtomicInteger(0); private Configuration config = HBaseConfiguration.create(); private boolean debug = false; private String tableName = ""; /** * A Cluster Connection instance that is shared by all running ycsb threads. * Needs to be initialized late so we pick up command-line configs if any. * To ensure one instance only in a multi-threaded context, guard access * with a 'lock' object. * @See #CONNECTION_LOCK. */ private static Connection connection = null; // Depending on the value of clientSideBuffering, either bufferedMutator // (clientSideBuffering) or currentTable (!clientSideBuffering) will be used. private Table currentTable = null; private BufferedMutator bufferedMutator = null; private String columnFamily = ""; private byte[] columnFamilyBytes; /** * Durability to use for puts and deletes. */ private Durability durability = Durability.USE_DEFAULT; /** Whether or not a page filter should be used to limit scan length. */ private boolean usePageFilter = true; /** * If true, buffer mutations on the client. This is the default behavior for * HBaseClient. For measuring insert/update/delete latencies, client side * buffering should be disabled. */ private boolean clientSideBuffering = false; private long writeBufferSize = 1024 * 1024 * 12; /** * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void init() throws DBException { if ("true" .equals(getProperties().getProperty("clientbuffering", "false"))) { this.clientSideBuffering = true; } if (getProperties().containsKey("writebuffersize")) { writeBufferSize = Long.parseLong(getProperties().getProperty("writebuffersize")); } if (getProperties().getProperty("durability") != null) { this.durability = Durability.valueOf(getProperties().getProperty("durability")); } if ("kerberos".equalsIgnoreCase(config.get("hbase.security.authentication"))) { config.set("hadoop.security.authentication", "Kerberos"); UserGroupInformation.setConfiguration(config); } if ((getProperties().getProperty("principal")!=null) && (getProperties().getProperty("keytab")!=null)) { try { UserGroupInformation.loginUserFromKeytab(getProperties().getProperty("principal"), getProperties().getProperty("keytab")); } catch (IOException e) { System.err.println("Keytab file is not readable or not found"); throw new DBException(e); } } String table = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); try { THREAD_COUNT.getAndIncrement(); synchronized (THREAD_COUNT) { if (connection == null) { // Initialize if not set up already. connection = ConnectionFactory.createConnection(config); // Terminate right now if table does not exist, since the client // will not propagate this error upstream once the workload // starts. final TableName tName = TableName.valueOf(table); connection.getTable(tName).getTableDescriptor(); } } } catch (java.io.IOException e) { throw new DBException(e); } if ((getProperties().getProperty("debug") != null) && (getProperties().getProperty("debug").compareTo("true") == 0)) { debug = true; } if ("false" .equals(getProperties().getProperty("hbase.usepagefilter", "true"))) { usePageFilter = false; } columnFamily = getProperties().getProperty("columnfamily"); if (columnFamily == null) { System.err.println("Error, must specify a columnfamily for HBase table"); throw new DBException("No columnfamily specified"); } columnFamilyBytes = Bytes.toBytes(columnFamily); } /** * Cleanup any state for this DB. Called once per DB instance; there is one DB * instance per client thread. */ @Override public void cleanup() throws DBException { // Get the measurements instance as this is the only client that should // count clean up time like an update if client-side buffering is // enabled. Measurements measurements = Measurements.getMeasurements(); try { long st = System.nanoTime(); if (bufferedMutator != null) { bufferedMutator.close(); } if (currentTable != null) { currentTable.close(); } long en = System.nanoTime(); final String type = clientSideBuffering ? "UPDATE" : "CLEANUP"; measurements.measure(type, (int) ((en - st) / 1000)); int threadCount = THREAD_COUNT.decrementAndGet(); if (threadCount <= 0) { // Means we are done so ok to shut down the Connection. synchronized (THREAD_COUNT) { if (connection != null) { connection.close(); connection = null; } } } } catch (IOException e) { throw new DBException(e); } } public void getHTable(String table) throws IOException { final TableName tName = TableName.valueOf(table); this.currentTable = connection.getTable(tName); if (clientSideBuffering) { final BufferedMutatorParams p = new BufferedMutatorParams(tName); p.writeBufferSize(writeBufferSize); this.bufferedMutator = connection.getBufferedMutator(p); } } /** * Read a record from the database. Each field/value pair from the result will * be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ public Status read(String table, String key, Set fields, Map result) { // if this is a "new" table, init HTable object. Else, use existing one if (!tableName.equals(table)) { currentTable = null; try { getHTable(table); tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase table: " + e); return Status.ERROR; } } Result r = null; try { if (debug) { System.out .println("Doing read from HBase columnfamily " + columnFamily); System.out.println("Doing read for key: " + key); } Get g = new Get(Bytes.toBytes(key)); if (fields == null) { g.addFamily(columnFamilyBytes); } else { for (String field : fields) { g.addColumn(columnFamilyBytes, Bytes.toBytes(field)); } } r = currentTable.get(g); } catch (IOException e) { if (debug) { System.err.println("Error doing get: " + e); } return Status.ERROR; } catch (ConcurrentModificationException e) { // do nothing for now...need to understand HBase concurrency model better return Status.ERROR; } if (r.isEmpty()) { return Status.NOT_FOUND; } while (r.advance()) { final Cell c = r.current(); result.put(Bytes.toString(CellUtil.cloneQualifier(c)), new ByteArrayByteIterator(CellUtil.cloneValue(c))); if (debug) { System.out.println( "Result for field: " + Bytes.toString(CellUtil.cloneQualifier(c)) + " is: " + Bytes.toString(CellUtil.cloneValue(c))); } } return Status.OK; } /** * Perform a range scan for a set of records in the database. Each field/value * pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // if this is a "new" table, init HTable object. Else, use existing one if (!tableName.equals(table)) { currentTable = null; try { getHTable(table); tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase table: " + e); return Status.ERROR; } } Scan s = new Scan(Bytes.toBytes(startkey)); // HBase has no record limit. Here, assume recordcount is small enough to // bring back in one call. // We get back recordcount records s.setCaching(recordcount); if (this.usePageFilter) { s.setFilter(new PageFilter(recordcount)); } // add specified fields or else all fields if (fields == null) { s.addFamily(columnFamilyBytes); } else { for (String field : fields) { s.addColumn(columnFamilyBytes, Bytes.toBytes(field)); } } // get results ResultScanner scanner = null; try { scanner = currentTable.getScanner(s); int numResults = 0; for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { // get row key String key = Bytes.toString(rr.getRow()); if (debug) { System.out.println("Got scan result for key: " + key); } HashMap rowResult = new HashMap(); while (rr.advance()) { final Cell cell = rr.current(); rowResult.put(Bytes.toString(CellUtil.cloneQualifier(cell)), new ByteArrayByteIterator(CellUtil.cloneValue(cell))); } // add rowResult to result vector result.add(rowResult); numResults++; // PageFilter does not guarantee that the number of results is <= // pageSize, so this // break is required. if (numResults >= recordcount) {// if hit recordcount, bail out break; } } // done with row } catch (IOException e) { if (debug) { System.out.println("Error in getting/parsing scan result: " + e); } return Status.ERROR; } finally { if (scanner != null) { scanner.close(); } } return Status.OK; } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error */ @Override public Status update(String table, String key, Map values) { // if this is a "new" table, init HTable object. Else, use existing one if (!tableName.equals(table)) { currentTable = null; try { getHTable(table); tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase table: " + e); return Status.ERROR; } } if (debug) { System.out.println("Setting up put for key: " + key); } Put p = new Put(Bytes.toBytes(key)); p.setDurability(durability); for (Map.Entry entry : values.entrySet()) { byte[] value = entry.getValue().toArray(); if (debug) { System.out.println("Adding field/value " + entry.getKey() + "/" + Bytes.toStringBinary(value) + " to put request"); } p.addColumn(columnFamilyBytes, Bytes.toBytes(entry.getKey()), value); } try { if (clientSideBuffering) { Preconditions.checkNotNull(bufferedMutator); bufferedMutator.mutate(p); } else { currentTable.put(p); } } catch (IOException e) { if (debug) { System.err.println("Error doing put: " + e); } return Status.ERROR; } catch (ConcurrentModificationException e) { // do nothing for now...hope this is rare return Status.ERROR; } return Status.OK; } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error */ @Override public Status insert(String table, String key, Map values) { return update(table, key, values); } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error */ @Override public Status delete(String table, String key) { // if this is a "new" table, init HTable object. Else, use existing one if (!tableName.equals(table)) { currentTable = null; try { getHTable(table); tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase table: " + e); return Status.ERROR; } } if (debug) { System.out.println("Doing delete for key: " + key); } final Delete d = new Delete(Bytes.toBytes(key)); d.setDurability(durability); try { if (clientSideBuffering) { Preconditions.checkNotNull(bufferedMutator); bufferedMutator.mutate(d); } else { currentTable.delete(d); } } catch (IOException e) { if (debug) { System.err.println("Error doing delete: " + e); } return Status.ERROR; } return Status.OK; } @VisibleForTesting void setConfiguration(final Configuration newConfig) { this.config = newConfig; } } /* * For customized vim control set autoindent set si set shiftwidth=4 */ ================================================ FILE: hbase10/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for HBase * using the HBase 1.0.0 API. */ package com.yahoo.ycsb.db; ================================================ FILE: hbase10/src/test/java/com/yahoo/ycsb/db/HBaseClient10Test.java ================================================ /** * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.workloads.CoreWorkload; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.List; import java.util.Properties; import java.util.Vector; /** * Integration tests for the YCSB HBase client 1.0, using an HBase minicluster. */ public class HBaseClient10Test { private final static String COLUMN_FAMILY = "cf"; private static HBaseTestingUtility testingUtil; private HBaseClient10 client; private Table table = null; private String tableName; private static boolean isWindows() { final String os = System.getProperty("os.name"); return os.startsWith("Windows"); } /** * Creates a mini-cluster for use in these tests. * * This is a heavy-weight operation, so invoked only once for the test class. */ @BeforeClass public static void setUpClass() throws Exception { // Minicluster setup fails on Windows with an UnsatisfiedLinkError. // Skip if windows. assumeTrue(!isWindows()); testingUtil = HBaseTestingUtility.createLocalHTU(); testingUtil.startMiniCluster(); } /** * Tears down mini-cluster. */ @AfterClass public static void tearDownClass() throws Exception { if (testingUtil != null) { testingUtil.shutdownMiniCluster(); } } /** * Sets up the mini-cluster for testing. * * We re-create the table for each test. */ @Before public void setUp() throws Exception { client = new HBaseClient10(); client.setConfiguration(new Configuration(testingUtil.getConfiguration())); Properties p = new Properties(); p.setProperty("columnfamily", COLUMN_FAMILY); Measurements.setProperties(p); final CoreWorkload workload = new CoreWorkload(); workload.init(p); tableName = p.getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); table = testingUtil.createTable(TableName.valueOf(tableName), Bytes.toBytes(COLUMN_FAMILY)); client.setProperties(p); client.init(); } @After public void tearDown() throws Exception { table.close(); testingUtil.deleteTable(tableName); } @Test public void testRead() throws Exception { final String rowKey = "row1"; final Put p = new Put(Bytes.toBytes(rowKey)); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes("column1"), Bytes.toBytes("value1")); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes("column2"), Bytes.toBytes("value2")); table.put(p); final HashMap result = new HashMap(); final Status status = client.read(tableName, rowKey, null, result); assertEquals(Status.OK, status); assertEquals(2, result.size()); assertEquals("value1", result.get("column1").toString()); assertEquals("value2", result.get("column2").toString()); } @Test public void testReadMissingRow() throws Exception { final HashMap result = new HashMap(); final Status status = client.read(tableName, "Missing row", null, result); assertEquals(Status.NOT_FOUND, status); assertEquals(0, result.size()); } @Test public void testScan() throws Exception { // Fill with data final String colStr = "row_number"; final byte[] col = Bytes.toBytes(colStr); final int n = 10; final List puts = new ArrayList(n); for(int i = 0; i < n; i++) { final byte[] key = Bytes.toBytes(String.format("%05d", i)); final byte[] value = java.nio.ByteBuffer.allocate(4).putInt(i).array(); final Put p = new Put(key); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), col, value); puts.add(p); } table.put(puts); // Test final Vector> result = new Vector>(); // Scan 5 records, skipping the first client.scan(tableName, "00001", 5, null, result); assertEquals(5, result.size()); for(int i = 0; i < 5; i++) { final Map row = result.get(i); assertEquals(1, row.size()); assertTrue(row.containsKey(colStr)); final byte[] bytes = row.get(colStr).toArray(); final ByteBuffer buf = ByteBuffer.wrap(bytes); final int rowNum = buf.getInt(); assertEquals(i + 1, rowNum); } } @Test public void testUpdate() throws Exception{ final String key = "key"; final Map input = new HashMap(); input.put("column1", "value1"); input.put("column2", "value2"); final Status status = client.insert(tableName, key, StringByteIterator.getByteIteratorMap(input)); assertEquals(Status.OK, status); // Verify result final Get get = new Get(Bytes.toBytes(key)); final Result result = this.table.get(get); assertFalse(result.isEmpty()); assertEquals(2, result.size()); for(final java.util.Map.Entry entry : input.entrySet()) { assertEquals(entry.getValue(), new String(result.getValue(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes(entry.getKey())))); } } @Test @Ignore("Not yet implemented") public void testDelete() { fail("Not yet implemented"); } } ================================================ FILE: hbase10/src/test/resources/hbase-site.xml ================================================ hbase.master.info.port -1 The port for the hbase master web UI Set to -1 if you do not want the info server to run. hbase.regionserver.info.port -1 The port for the hbase regionserver web UI Set to -1 if you do not want the info server to run. ================================================ FILE: hbase10/src/test/resources/log4j.properties ================================================ # # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=WARN, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZKTableStateManager: Creates a large number of table # state change messages. log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKTableStateManager=ERROR ================================================ FILE: hbase12/README.md ================================================ # HBase (1.2+) Driver for YCSB This driver is a binding for the YCSB facilities to operate against a HBase 1.2+ Server cluster, using a shaded client that tries to avoid leaking third party libraries. See `hbase098/README.md` for a quickstart to setup HBase for load testing and common configuration details. ## Configuration Options In addition to those options available for the `hbase098` binding, the following options are available for the `hbase12` binding: * `durability`: Whether or not writes should be appended to the WAL. Bypassing the WAL can improve throughput but data cannot be recovered in the event of a crash. The default is true. ================================================ FILE: hbase12/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ hbase12-binding HBase 1.2 DB Binding true true com.yahoo.ycsb hbase10-binding ${project.version} org.apache.hbase hbase-client com.yahoo.ycsb core ${project.version} provided org.apache.hbase hbase-shaded-client ${hbase12.version} junit junit 4.12 test ================================================ FILE: hbase12/src/main/java/com/yahoo/ycsb/db/hbase12/HBaseClient12.java ================================================ /** * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.hbase12; /** * HBase 1.2 client for YCSB framework. * * A modified version of HBaseClient (which targets HBase v1.2) utilizing the * shaded client. * * It should run equivalent to following the hbase098 binding README. * */ public class HBaseClient12 extends com.yahoo.ycsb.db.HBaseClient10 { } ================================================ FILE: hbase12/src/main/java/com/yahoo/ycsb/db/hbase12/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for HBase * using the HBase 1.2+ shaded API. */ package com.yahoo.ycsb.db.hbase12; ================================================ FILE: hbase12/src/test/java/com/yahoo/ycsb/db/hbase12/HBaseClient12Test.java ================================================ /** * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.hbase12; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.measurements.Measurements; import com.yahoo.ycsb.workloads.CoreWorkload; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Properties; import java.util.Vector; /** * Integration tests for the YCSB HBase client 1.2, using an HBase minicluster. */ public class HBaseClient12Test { private final static String COLUMN_FAMILY = "cf"; private static HBaseTestingUtility testingUtil; private HBaseClient12 client; private Table table = null; private String tableName; private static boolean isWindows() { final String os = System.getProperty("os.name"); return os.startsWith("Windows"); } /** * Creates a mini-cluster for use in these tests. * * This is a heavy-weight operation, so invoked only once for the test class. */ @BeforeClass public static void setUpClass() throws Exception { // Minicluster setup fails on Windows with an UnsatisfiedLinkError. // Skip if windows. assumeTrue(!isWindows()); testingUtil = HBaseTestingUtility.createLocalHTU(); testingUtil.startMiniCluster(); } /** * Tears down mini-cluster. */ @AfterClass public static void tearDownClass() throws Exception { if (testingUtil != null) { testingUtil.shutdownMiniCluster(); } } /** * Sets up the mini-cluster for testing. * * We re-create the table for each test. */ @Before public void setUp() throws Exception { client = new HBaseClient12(); client.setConfiguration(new Configuration(testingUtil.getConfiguration())); Properties p = new Properties(); p.setProperty("columnfamily", COLUMN_FAMILY); Measurements.setProperties(p); final CoreWorkload workload = new CoreWorkload(); workload.init(p); tableName = p.getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); table = testingUtil.createTable(TableName.valueOf(tableName), Bytes.toBytes(COLUMN_FAMILY)); client.setProperties(p); client.init(); } @After public void tearDown() throws Exception { table.close(); testingUtil.deleteTable(tableName); } @Test public void testRead() throws Exception { final String rowKey = "row1"; final Put p = new Put(Bytes.toBytes(rowKey)); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes("column1"), Bytes.toBytes("value1")); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes("column2"), Bytes.toBytes("value2")); table.put(p); final HashMap result = new HashMap(); final Status status = client.read(tableName, rowKey, null, result); assertEquals(Status.OK, status); assertEquals(2, result.size()); assertEquals("value1", result.get("column1").toString()); assertEquals("value2", result.get("column2").toString()); } @Test public void testReadMissingRow() throws Exception { final HashMap result = new HashMap(); final Status status = client.read(tableName, "Missing row", null, result); assertEquals(Status.NOT_FOUND, status); assertEquals(0, result.size()); } @Test public void testScan() throws Exception { // Fill with data final String colStr = "row_number"; final byte[] col = Bytes.toBytes(colStr); final int n = 10; final List puts = new ArrayList(n); for(int i = 0; i < n; i++) { final byte[] key = Bytes.toBytes(String.format("%05d", i)); final byte[] value = java.nio.ByteBuffer.allocate(4).putInt(i).array(); final Put p = new Put(key); p.addColumn(Bytes.toBytes(COLUMN_FAMILY), col, value); puts.add(p); } table.put(puts); // Test final Vector> result = new Vector>(); // Scan 5 records, skipping the first client.scan(tableName, "00001", 5, null, result); assertEquals(5, result.size()); for(int i = 0; i < 5; i++) { final HashMap row = result.get(i); assertEquals(1, row.size()); assertTrue(row.containsKey(colStr)); final byte[] bytes = row.get(colStr).toArray(); final ByteBuffer buf = ByteBuffer.wrap(bytes); final int rowNum = buf.getInt(); assertEquals(i + 1, rowNum); } } @Test public void testUpdate() throws Exception{ final String key = "key"; final HashMap input = new HashMap(); input.put("column1", "value1"); input.put("column2", "value2"); final Status status = client.insert(tableName, key, StringByteIterator.getByteIteratorMap(input)); assertEquals(Status.OK, status); // Verify result final Get get = new Get(Bytes.toBytes(key)); final Result result = this.table.get(get); assertFalse(result.isEmpty()); assertEquals(2, result.size()); for(final java.util.Map.Entry entry : input.entrySet()) { assertEquals(entry.getValue(), new String(result.getValue(Bytes.toBytes(COLUMN_FAMILY), Bytes.toBytes(entry.getKey())))); } } @Test @Ignore("Not yet implemented") public void testDelete() { fail("Not yet implemented"); } } ================================================ FILE: hbase12/src/test/resources/hbase-site.xml ================================================ hbase.master.info.port -1 The port for the hbase master web UI Set to -1 if you do not want the info server to run. hbase.regionserver.info.port -1 The port for the hbase regionserver web UI Set to -1 if you do not want the info server to run. ================================================ FILE: hbase12/src/test/resources/log4j.properties ================================================ # # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=WARN, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZKTableStateManager: Creates a large number of table # state change messages. log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKTableStateManager=ERROR ================================================ FILE: hypertable/README.md ================================================ # Install Hypertable Installation instructions for Hypertable can be found at: code.google.com/p/hypertable/wiki/HypertableManual # Set Up YCSB Clone the YCSB git repository and compile: ]$ git clone git://github.com/brianfrankcooper/YCSB.git ]$ cd YCSB ]$ mvn clean package # Run Hypertable Once it has been installed, start Hypertable by running ]$ ./bin/ht start all-servers hadoop if an instance of HDFS is running or ]$ ./bin/ht start all-servers local if the database is backed by the local file system. YCSB accesses a table called 'usertable' by default. Create this table through the Hypertable shell by running ]$ ./bin/ht shell hypertable> use '/ycsb'; hypertable> create table usertable(family); hypertable> quit All iteractions by YCSB take place under the Hypertable namespace '/ycsb'. Hypertable also uses an additional data grouping structure called a column family that must be set. YCSB doesn't offer fine grained operations on column families so in this example the table is created with a single column family named 'family' to which all column families will belong. The name of this column family must be passed to YCSB. The table can be manipulated from within the hypertable shell without interfering with the operation of YCSB. # Run YCSB Make sure that an instance of Hypertable is running. To access the database through the YCSB shell, from the YCSB directory run: ]$ ./bin/ycsb shell hypertable -p columnfamily=family where the value passed to columnfamily matches that used in the table creation. To run a workload, first load the data: ]$ ./bin/ycsb load hypertable -P workloads/workloada -p columnfamily=family Then run the workload: ]$ ./bin/ycsb run hypertable -P workloads/workloada -p columnfamily=family This example runs the core workload 'workloada' that comes packaged with YCSB. The state of the YCSB data in the Hypertable database can be reset by dropping usertable and recreating it. # Configuration Parameters Hypertable configuration settings can be found in conf/hypertable.cfg under your main hypertable directory. Make sure that the constant THRIFTBROKER_PORT in the class HypertableClient matches the setting ThriftBroker.Port in hypertable.cfg. To change the amount of data returned on each call to the ThriftClient on a Hypertable scan, one must add a new parameter to hypertable.cfg. Include ThriftBroker.NextThreshold=x where x is set to the size desired in bytes. The default setting of this parameter is 128000. To alter the Hypertable namespace YCSB operates under, change the constant NAMESPACE in the class HypertableClient. ================================================ FILE: hypertable/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent hypertable-binding Hypertable DB Binding jar com.yahoo.ycsb core ${project.version} provided org.apache.thrift libthrift ${thrift.version} org.hypertable hypertable ${hypertable.version} clojars.org http://clojars.org/repo ================================================ FILE: hypertable/src/main/java/com/yahoo/ycsb/db/HypertableClient.java ================================================ /** * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import org.apache.thrift.TException; import org.hypertable.thrift.SerializedCellsFlag; import org.hypertable.thrift.SerializedCellsReader; import org.hypertable.thrift.SerializedCellsWriter; import org.hypertable.thrift.ThriftClient; import org.hypertable.thriftgen.Cell; import org.hypertable.thriftgen.ClientException; import org.hypertable.thriftgen.Key; import org.hypertable.thriftgen.KeyFlag; import org.hypertable.thriftgen.RowInterval; import org.hypertable.thriftgen.ScanSpec; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; /** * Hypertable client for YCSB framework. */ public class HypertableClient extends com.yahoo.ycsb.DB { public static final String NAMESPACE = "/ycsb"; public static final int THRIFTBROKER_PORT = 38080; public static final int BUFFER_SIZE = 4096; private boolean debug = false; private ThriftClient connection; private long ns; private String columnFamily = ""; /** * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void init() throws DBException { if ((getProperties().getProperty("debug") != null) && (getProperties().getProperty("debug").equals("true"))) { debug = true; } try { connection = ThriftClient.create("localhost", THRIFTBROKER_PORT); if (!connection.namespace_exists(NAMESPACE)) { connection.namespace_create(NAMESPACE); } ns = connection.open_namespace(NAMESPACE); } catch (ClientException e) { throw new DBException("Could not open namespace", e); } catch (TException e) { throw new DBException("Could not open namespace", e); } columnFamily = getProperties().getProperty("columnfamily"); if (columnFamily == null) { System.err.println( "Error, must specify a " + "columnfamily for Hypertable table"); throw new DBException("No columnfamily specified"); } } /** * Cleanup any state for this DB. Called once per DB instance; there is one DB * instance per client thread. */ @Override public void cleanup() throws DBException { try { connection.namespace_close(ns); } catch (ClientException e) { throw new DBException("Could not close namespace", e); } catch (TException e) { throw new DBException("Could not close namespace", e); } } /** * Read a record from the database. Each field/value pair from the result will * be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ @Override public Status read(String table, String key, Set fields, Map result) { // SELECT _column_family:field[i] // FROM table WHERE ROW=key MAX_VERSIONS 1; if (debug) { System.out .println("Doing read from Hypertable columnfamily " + columnFamily); System.out.println("Doing read for key: " + key); } try { if (null != fields) { Vector> resMap = new Vector>(); if (!scan(table, key, 1, fields, resMap).equals(Status.OK)) { return Status.ERROR; } if (!resMap.isEmpty()) { result.putAll(resMap.firstElement()); } } else { SerializedCellsReader reader = new SerializedCellsReader(null); reader.reset(connection.get_row_serialized(ns, table, key)); while (reader.next()) { result.put(new String(reader.get_column_qualifier()), new ByteArrayByteIterator(reader.get_value())); } } } catch (ClientException e) { if (debug) { System.err.println("Error doing read: " + e.message); } return Status.ERROR; } catch (TException e) { if (debug) { System.err.println("Error doing read"); } return Status.ERROR; } return Status.OK; } /** * Perform a range scan for a set of records in the database. Each field/value * pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { // SELECT _columnFamily:fields FROM table WHERE (ROW >= startkey) // LIMIT recordcount MAX_VERSIONS 1; ScanSpec spec = new ScanSpec(); RowInterval elem = new RowInterval(); elem.setStart_inclusive(true); elem.setStart_row(startkey); spec.addToRow_intervals(elem); if (null != fields) { for (String field : fields) { spec.addToColumns(columnFamily + ":" + field); } } spec.setVersions(1); spec.setRow_limit(recordcount); SerializedCellsReader reader = new SerializedCellsReader(null); try { long sc = connection.scanner_open(ns, table, spec); String lastRow = null; boolean eos = false; while (!eos) { reader.reset(connection.scanner_get_cells_serialized(sc)); while (reader.next()) { String currentRow = new String(reader.get_row()); if (!currentRow.equals(lastRow)) { result.add(new HashMap()); lastRow = currentRow; } result.lastElement().put(new String(reader.get_column_qualifier()), new ByteArrayByteIterator(reader.get_value())); } eos = reader.eos(); if (debug) { System.out .println("Number of rows retrieved so far: " + result.size()); } } connection.scanner_close(sc); } catch (ClientException e) { if (debug) { System.err.println("Error doing scan: " + e.message); } return Status.ERROR; } catch (TException e) { if (debug) { System.err.println("Error doing scan"); } return Status.ERROR; } return Status.OK; } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error */ @Override public Status update(String table, String key, Map values) { return insert(table, key, values); } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error */ @Override public Status insert(String table, String key, Map values) { // INSERT INTO table VALUES // (key, _column_family:entry,getKey(), entry.getValue()), (...); if (debug) { System.out.println("Setting up put for key: " + key); } try { long mutator = connection.mutator_open(ns, table, 0, 0); SerializedCellsWriter writer = new SerializedCellsWriter(BUFFER_SIZE * values.size(), true); for (Map.Entry entry : values.entrySet()) { writer.add(key, columnFamily, entry.getKey(), SerializedCellsFlag.AUTO_ASSIGN, ByteBuffer.wrap(entry.getValue().toArray())); } connection.mutator_set_cells_serialized(mutator, writer.buffer(), true); connection.mutator_close(mutator); } catch (ClientException e) { if (debug) { System.err.println("Error doing set: " + e.message); } return Status.ERROR; } catch (TException e) { if (debug) { System.err.println("Error doing set"); } return Status.ERROR; } return Status.OK; } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error */ @Override public Status delete(String table, String key) { // DELETE * FROM table WHERE ROW=key; if (debug) { System.out.println("Doing delete for key: " + key); } Cell entry = new Cell(); entry.key = new Key(); entry.key.row = key; entry.key.flag = KeyFlag.DELETE_ROW; try { connection.set_cell(ns, table, entry); } catch (ClientException e) { if (debug) { System.err.println("Error doing delete: " + e.message); } return Status.ERROR; } catch (TException e) { if (debug) { System.err.println("Error doing delete"); } return Status.ERROR; } return Status.OK; } } ================================================ FILE: hypertable/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Hypertable. */ package com.yahoo.ycsb.db; ================================================ FILE: infinispan/README.md ================================================ ## Quick Start This section describes how to run YCSB on infinispan. ### 1. Install Java and Maven ### 2. Set Up YCSB 1. Git clone YCSB and compile: ``` git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ``` 2. Copy and untar YCSB distribution in distribution/target/ycsb-x.x.x.tar.gz to target machine ### 4. Load data and run tests ####4.1 embedded mode with cluster or not Load the data: ``` ./bin/ycsb load infinispan -P workloads/workloada -p infinispan.clustered= ``` Run the workload test: ``` ./bin/ycsb run infinispan -s -P workloads/workloada -p infinispan.clustered= ``` ####4.2 client-server mode 1. start infinispan server 2. read [RemoteCacheManager](http://docs.jboss.org/infinispan/7.2/apidocs/org/infinispan/client/hotrod/RemoteCacheManager.html) doc and customize hotrod client properties in infinispan-binding/conf/remote-cache.properties 3. Load the data with specified cache: ``` ./bin/ycsb load infinispan-cs -s -P workloads/workloada -P infinispan-binding/conf/remote-cache.properties -p cache= ``` 4. Run the workload test with specified cache: ``` ./bin/ycsb run infinispan-cs -s -P workloads/workloada -P infinispan-binding/conf/remote-cache.properties -p cache= ``` ================================================ FILE: infinispan/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent infinispan-binding Infinispan DB Binding jar org.infinispan infinispan-client-hotrod ${infinispan.version} org.infinispan infinispan-core ${infinispan.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: infinispan/src/main/conf/infinispan-config.xml ================================================ ================================================ FILE: infinispan/src/main/conf/remote-cache.properties ================================================ # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. infinispan.client.hotrod.server_list=192.168.101.17:11222 infinispan.client.hotrod.force_return_values=false maxActive=-1 maxIdle=-1 minIdle=1 maxTotal=-1 whenExhaustedAction=1 ================================================ FILE: infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanClient.java ================================================ /** * Copyright (c) 2012-2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.infinispan.Cache; import org.infinispan.atomic.AtomicMap; import org.infinispan.atomic.AtomicMapLookup; import org.infinispan.manager.DefaultCacheManager; import org.infinispan.manager.EmbeddedCacheManager; import org.infinispan.util.logging.Log; import org.infinispan.util.logging.LogFactory; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; /** * This is a client implementation for Infinispan 5.x. */ public class InfinispanClient extends DB { private static final Log LOGGER = LogFactory.getLog(InfinispanClient.class); // An optimisation for clustered mode private final boolean clustered; private EmbeddedCacheManager infinispanManager; public InfinispanClient() { clustered = Boolean.getBoolean("infinispan.clustered"); } public void init() throws DBException { try { infinispanManager = new DefaultCacheManager("infinispan-config.xml"); } catch (IOException e) { throw new DBException(e); } } public void cleanup() { infinispanManager.stop(); infinispanManager = null; } public Status read(String table, String key, Set fields, Map result) { try { Map row; if (clustered) { row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key, false); } else { Cache> cache = infinispanManager.getCache(table); row = cache.get(key); } if (row != null) { result.clear(); if (fields == null || fields.isEmpty()) { StringByteIterator.putAllAsByteIterators(result, row); } else { for (String field : fields) { result.put(field, new StringByteIterator(row.get(field))); } } } return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { LOGGER.warn("Infinispan does not support scan semantics"); return Status.OK; } public Status update(String table, String key, Map values) { try { if (clustered) { AtomicMap row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key); StringByteIterator.putAllAsStrings(row, values); } else { Cache> cache = infinispanManager.getCache(table); Map row = cache.get(key); if (row == null) { row = StringByteIterator.getStringMap(values); cache.put(key, row); } else { StringByteIterator.putAllAsStrings(row, values); } } return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } public Status insert(String table, String key, Map values) { try { if (clustered) { AtomicMap row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key); row.clear(); StringByteIterator.putAllAsStrings(row, values); } else { infinispanManager.getCache(table).put(key, values); } return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } public Status delete(String table, String key) { try { if (clustered) { AtomicMapLookup.removeAtomicMap(infinispanManager.getCache(table), key); } else { infinispanManager.getCache(table).remove(key); } return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } } ================================================ FILE: infinispan/src/main/java/com/yahoo/ycsb/db/InfinispanRemoteClient.java ================================================ /** * Copyright (c) 2015-2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.*; import org.infinispan.client.hotrod.RemoteCache; import org.infinispan.client.hotrod.RemoteCacheManager; import org.infinispan.util.logging.Log; import org.infinispan.util.logging.LogFactory; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; /** * This is a client implementation for Infinispan 5.x in client-server mode. */ public class InfinispanRemoteClient extends DB { private static final Log LOGGER = LogFactory.getLog(InfinispanRemoteClient.class); private RemoteCacheManager remoteIspnManager; private String cacheName = null; @Override public void init() throws DBException { remoteIspnManager = RemoteCacheManagerHolder.getInstance(getProperties()); cacheName = getProperties().getProperty("cache"); } @Override public void cleanup() { remoteIspnManager.stop(); remoteIspnManager = null; } @Override public Status insert(String table, String recordKey, Map values) { String compositKey = createKey(table, recordKey); Map stringValues = new HashMap<>(); StringByteIterator.putAllAsStrings(stringValues, values); try { cache().put(compositKey, stringValues); return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } @Override public Status read(String table, String recordKey, Set fields, Map result) { String compositKey = createKey(table, recordKey); try { Map values = cache().get(compositKey); if (values == null || values.isEmpty()) { return Status.NOT_FOUND; } if (fields == null) { //get all field/value pairs StringByteIterator.putAllAsByteIterators(result, values); } else { for (String field : fields) { String value = values.get(field); if (value != null) { result.put(field, new StringByteIterator(value)); } } } return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { LOGGER.warn("Infinispan does not support scan semantics"); return Status.NOT_IMPLEMENTED; } @Override public Status update(String table, String recordKey, Map values) { String compositKey = createKey(table, recordKey); try { Map stringValues = new HashMap<>(); StringByteIterator.putAllAsStrings(stringValues, values); cache().put(compositKey, stringValues); return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } @Override public Status delete(String table, String recordKey) { String compositKey = createKey(table, recordKey); try { cache().remove(compositKey); return Status.OK; } catch (Exception e) { LOGGER.error(e); return Status.ERROR; } } private RemoteCache> cache() { if (this.cacheName != null) { return remoteIspnManager.getCache(cacheName); } else { return remoteIspnManager.getCache(); } } private String createKey(String table, String recordKey) { return table + "-" + recordKey; } } ================================================ FILE: infinispan/src/main/java/com/yahoo/ycsb/db/RemoteCacheManagerHolder.java ================================================ /** * Copyright (c) 2015-2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.util.Properties; import org.infinispan.client.hotrod.RemoteCacheManager; /** * Utility class to ensure only a single RemoteCacheManager is created. */ final class RemoteCacheManagerHolder { private static volatile RemoteCacheManager cacheManager = null; private RemoteCacheManagerHolder() { } static RemoteCacheManager getInstance(Properties props) { RemoteCacheManager result = cacheManager; if (result == null) { synchronized (RemoteCacheManagerHolder.class) { result = cacheManager; if (result == null) { result = new RemoteCacheManager(props); cacheManager = new RemoteCacheManager(props); } } } return result; } } ================================================ FILE: infinispan/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2015-2016 YCSB Contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Infinispan. */ package com.yahoo.ycsb.db; ================================================ FILE: jdbc/README.md ================================================ # JDBC Driver for YCSB This driver enables YCSB to work with databases accessible via the JDBC protocol. ## Getting Started ### 1. Start your database This driver will connect to databases that use the JDBC protocol, please refer to your databases documentation on information on how to install, configure and start your system. ### 2. Set up YCSB You can clone the YCSB project and compile it to stay up to date with the latest changes. Or you can just download the latest release and unpack it. Either way, instructions for doing so can be found here: https://github.com/brianfrankcooper/YCSB. ### 3. Configure your database and table. You can name your database what ever you want, you will need to provide the database name in the JDBC connection string. You can name your table whatever you like also, but it needs to be specified using the YCSB core properties, the default is to just use 'usertable' as the table name. The expected table schema will look similar to the following, syntactical differences may exist with your specific database: ```sql CREATE TABLE usertable ( YCSB_KEY VARCHAR(255) PRIMARY KEY, FIELD0 TEXT, FIELD1 TEXT, FIELD2 TEXT, FIELD3 TEXT, FIELD4 TEXT, FIELD5 TEXT, FIELD6 TEXT, FIELD7 TEXT, FIELD8 TEXT, FIELD9 TEXT ); ``` Key take aways: * The primary key field needs to be named YCSB_KEY * The other fields need to be prefixed with FIELD and count up starting from 1 * Add the same number of FIELDs as you specify in the YCSB core properties, default is 10. * The type of the fields is not so important as long as they can accept strings of the length that you specify in the YCSB core properties, default is 100. #### JdbcDBCreateTable Utility YCSB has a utility to help create your SQL table. NOTE: It does not support all databases flavors, if it does not work for you, you will have to create your table manually with the schema given above. An example usage of the utility: ```sh java -cp YCSB_HOME/jdbc-binding/lib/jdbc-binding-0.4.0.jar:mysql-connector-java-5.1.37-bin.jar com.yahoo.ycsb.db.JdbcDBCreateTable -P db.properties -n usertable ``` Hint: you need to include your Driver jar in the classpath as well as specify JDBC connection information via a properties file, and a table name with ```-n```. Simply executing the JdbcDBCreateTable class without any other parameters will print out usage information. ### 4. Configure YCSB connection properties You need to set the following connection configurations: ```sh db.driver=com.mysql.jdbc.Driver db.url=jdbc:mysql://127.0.0.1:3306/ycsb db.user=admin db.passwd=admin ``` Be sure to use your driver class, a valid JDBC connection string, and credentials to your database. You can add these to your workload configuration or a separate properties file and specify it with ```-P``` or you can add the properties individually to your ycsb command with ```-p```. ### 5. Add your JDBC Driver to the classpath There are several ways to do this, but a couple easy methods are to put a copy of your Driver jar in ```YCSB_HOME/jdbc-binding/lib/``` or just specify the path to your Driver jar with ```-cp``` in your ycsb command. ### 6. Running a workload Before you can actually run the workload, you need to "load" the data first. ```sh bin/ycsb load jdbc -P workloads/workloada -P db.properties -cp mysql-connector-java.jar ``` Then, you can run the workload: ```sh bin/ycsb run jdbc -P workloads/workloada -P db.properties -cp mysql-connector-java.jar ``` ## Configuration Properties ```sh db.driver=com.mysql.jdbc.Driver # The JDBC driver class to use. db.url=jdbc:mysql://127.0.0.1:3306/ycsb # The Database connection URL. db.user=admin # User name for the connection. db.passwd=admin # Password for the connection. db.batchsize=1000 # The batch size for doing batched inserts. Defaults to 0. Set to >0 to use batching. jdbc.fetchsize=10 # The JDBC fetch size hinted to the driver. jdbc.autocommit=true # The JDBC connection auto-commit property for the driver. jdbc.batchupdateapi=false # Use addBatch()/executeBatch() JDBC methods instead of executeUpdate() for writes (default: false) db.batchsize=1000 # The number of rows to be batched before commit (or executeBatch() when jdbc.batchupdateapi=true) ``` Please refer to https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for all other YCSB core properties. ================================================ FILE: jdbc/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent jdbc-binding JDBC DB Binding jar org.apache.openjpa openjpa-jdbc ${openjpa.jdbc.version} com.yahoo.ycsb core ${project.version} provided junit junit 4.12 test org.hsqldb hsqldb 2.3.3 test ================================================ FILE: jdbc/src/main/conf/db.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Properties file that contains database connection information. db.driver=org.h2.Driver # jdbc.fetchsize=20 db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb db.user=sa db.passwd= ================================================ FILE: jdbc/src/main/conf/h2.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Properties file that contains database connection information. db.driver=org.h2.Driver db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb db.user=sa db.passwd= ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBCli.java ================================================ /** * Copyright (c) 2010 - 2016 Yahoo! Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.io.FileInputStream; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Enumeration; import java.util.Properties; /** * Execute a JDBC command line. * * @author sudipto */ public final class JdbcDBCli { private static void usageMessage() { System.out.println("JdbcCli. Options:"); System.out.println(" -p key=value properties defined."); System.out.println(" -P location of the properties file to load."); System.out.println(" -c SQL command to execute."); } private static void executeCommand(Properties props, String sql) throws SQLException { String driver = props.getProperty(JdbcDBClient.DRIVER_CLASS); String username = props.getProperty(JdbcDBClient.CONNECTION_USER); String password = props.getProperty(JdbcDBClient.CONNECTION_PASSWD, ""); String url = props.getProperty(JdbcDBClient.CONNECTION_URL); if (driver == null || username == null || url == null) { throw new SQLException("Missing connection information."); } Connection conn = null; try { Class.forName(driver); conn = DriverManager.getConnection(url, username, password); Statement stmt = conn.createStatement(); stmt.execute(sql); System.out.println("Command \"" + sql + "\" successfully executed."); } catch (ClassNotFoundException e) { throw new SQLException("JDBC Driver class not found."); } finally { if (conn != null) { System.out.println("Closing database connection."); conn.close(); } } } /** * @param args */ public static void main(String[] args) { if (args.length == 0) { usageMessage(); System.exit(0); } Properties props = new Properties(); Properties fileprops = new Properties(); String sql = null; // parse arguments int argindex = 0; while (args[argindex].startsWith("-")) { if (args[argindex].compareTo("-P") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } String propfile = args[argindex]; argindex++; Properties myfileprops = new Properties(); try { myfileprops.load(new FileInputStream(propfile)); } catch (IOException e) { System.out.println(e.getMessage()); System.exit(0); } // Issue #5 - remove call to stringPropertyNames to make compilable // under Java 1.5 for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, myfileprops.getProperty(prop)); } } else if (args[argindex].compareTo("-p") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } int eq = args[argindex].indexOf('='); if (eq < 0) { usageMessage(); System.exit(0); } String name = args[argindex].substring(0, eq); String value = args[argindex].substring(eq + 1); props.put(name, value); argindex++; } else if (args[argindex].compareTo("-c") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } sql = args[argindex++]; } else { System.out.println("Unknown option " + args[argindex]); usageMessage(); System.exit(0); } if (argindex >= args.length) { break; } } if (argindex != args.length) { usageMessage(); System.exit(0); } // overwrite file properties with properties from the command line // Issue #5 - remove call to stringPropertyNames to make compilable under // Java 1.5 for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, props.getProperty(prop)); } if (sql == null) { System.err.println("Missing command."); usageMessage(); System.exit(1); } try { executeCommand(fileprops, sql); } catch (SQLException e) { System.err.println("Error in executing command. " + e); System.exit(1); } } /** * Hidden constructor. */ private JdbcDBCli() { super(); } } ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClient.java ================================================ /** * Copyright (c) 2010 - 2016 Yahoo! Inc., 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import java.sql.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import com.yahoo.ycsb.db.flavors.DBFlavor; /** * A class that wraps a JDBC compliant database to allow it to be interfaced * with YCSB. This class extends {@link DB} and implements the database * interface used by YCSB client. * *
* Each client will have its own instance of this class. This client is not * thread safe. * *
* This interface expects a schema ... All * attributes are of type TEXT. All accesses are through the primary key. * Therefore, only one index on the primary key is needed. */ public class JdbcDBClient extends DB { /** The class to use as the jdbc driver. */ public static final String DRIVER_CLASS = "db.driver"; /** The URL to connect to the database. */ public static final String CONNECTION_URL = "db.url"; /** The user name to use to connect to the database. */ public static final String CONNECTION_USER = "db.user"; /** The password to use for establishing the connection. */ public static final String CONNECTION_PASSWD = "db.passwd"; /** The batch size for batched inserts. Set to >0 to use batching */ public static final String DB_BATCH_SIZE = "db.batchsize"; /** The JDBC fetch size hinted to the driver. */ public static final String JDBC_FETCH_SIZE = "jdbc.fetchsize"; /** The JDBC connection auto-commit property for the driver. */ public static final String JDBC_AUTO_COMMIT = "jdbc.autocommit"; public static final String JDBC_BATCH_UPDATES = "jdbc.batchupdateapi"; /** The name of the property for the number of fields in a record. */ public static final String FIELD_COUNT_PROPERTY = "fieldcount"; /** Default number of fields in a record. */ public static final String FIELD_COUNT_PROPERTY_DEFAULT = "10"; /** Representing a NULL value. */ public static final String NULL_VALUE = "NULL"; /** The primary key in the user table. */ public static final String PRIMARY_KEY = "YCSB_KEY"; /** The field name prefix in the table. */ public static final String COLUMN_PREFIX = "FIELD"; private List conns; private boolean initialized = false; private Properties props; private int jdbcFetchSize; private int batchSize; private boolean autoCommit; private boolean batchUpdates; private static final String DEFAULT_PROP = ""; private ConcurrentMap cachedStatements; private long numRowsInBatch = 0; /** DB flavor defines DB-specific syntax and behavior for the * particular database. Current database flavors are: {default, phoenix} */ private DBFlavor dbFlavor; /** * Ordered field information for insert and update statements. */ private static class OrderedFieldInfo { private String fieldKeys; private List fieldValues; OrderedFieldInfo(String fieldKeys, List fieldValues) { this.fieldKeys = fieldKeys; this.fieldValues = fieldValues; } String getFieldKeys() { return fieldKeys; } List getFieldValues() { return fieldValues; } } /** * For the given key, returns what shard contains data for this key. * * @param key Data key to do operation on * @return Shard index */ private int getShardIndexByKey(String key) { int ret = Math.abs(key.hashCode()) % conns.size(); return ret; } /** * For the given key, returns Connection object that holds connection to the * shard that contains this key. * * @param key Data key to get information for * @return Connection object */ private Connection getShardConnectionByKey(String key) { return conns.get(getShardIndexByKey(key)); } private void cleanupAllConnections() throws SQLException { for (Connection conn : conns) { if (!autoCommit) { conn.commit(); } conn.close(); } } /** Returns parsed int value from the properties if set, otherwise returns -1. */ private static int getIntProperty(Properties props, String key) throws DBException { String valueStr = props.getProperty(key); if (valueStr != null) { try { return Integer.parseInt(valueStr); } catch (NumberFormatException nfe) { System.err.println("Invalid " + key + " specified: " + valueStr); throw new DBException(nfe); } } return -1; } /** Returns parsed boolean value from the properties if set, otherwise returns defaultVal. */ private static boolean getBoolProperty(Properties props, String key, boolean defaultVal) { String valueStr = props.getProperty(key); if (valueStr != null) { return Boolean.parseBoolean(valueStr); } return defaultVal; } @Override public void init() throws DBException { if (initialized) { System.err.println("Client connection already initialized."); return; } props = getProperties(); String urls = props.getProperty(CONNECTION_URL, DEFAULT_PROP); String user = props.getProperty(CONNECTION_USER, DEFAULT_PROP); String passwd = props.getProperty(CONNECTION_PASSWD, DEFAULT_PROP); String driver = props.getProperty(DRIVER_CLASS); this.jdbcFetchSize = getIntProperty(props, JDBC_FETCH_SIZE); this.batchSize = getIntProperty(props, DB_BATCH_SIZE); this.autoCommit = getBoolProperty(props, JDBC_AUTO_COMMIT, true); this.batchUpdates = getBoolProperty(props, JDBC_BATCH_UPDATES, false); try { if (driver != null) { Class.forName(driver); } int shardCount = 0; conns = new ArrayList(3); final String[] urlArr = urls.split(","); for (String url : urlArr) { System.out.println("Adding shard node URL: " + url); Connection conn = DriverManager.getConnection(url, user, passwd); // Since there is no explicit commit method in the DB interface, all // operations should auto commit, except when explicitly told not to // (this is necessary in cases such as for PostgreSQL when running a // scan workload with fetchSize) conn.setAutoCommit(autoCommit); shardCount++; conns.add(conn); } System.out.println("Using shards: " + shardCount + ", batchSize:" + batchSize + ", fetchSize: " + jdbcFetchSize); cachedStatements = new ConcurrentHashMap(); this.dbFlavor = DBFlavor.fromJdbcUrl(urlArr[0]); } catch (ClassNotFoundException e) { System.err.println("Error in initializing the JDBS driver: " + e); throw new DBException(e); } catch (SQLException e) { System.err.println("Error in database operation: " + e); throw new DBException(e); } catch (NumberFormatException e) { System.err.println("Invalid value for fieldcount property. " + e); throw new DBException(e); } initialized = true; } @Override public void cleanup() throws DBException { if (batchSize > 0) { try { // commit un-finished batches for (PreparedStatement st : cachedStatements.values()) { if (!st.getConnection().isClosed() && !st.isClosed() && (numRowsInBatch % batchSize != 0)) { st.executeBatch(); } } } catch (SQLException e) { System.err.println("Error in cleanup execution. " + e); throw new DBException(e); } } try { cleanupAllConnections(); } catch (SQLException e) { System.err.println("Error in closing the connection. " + e); throw new DBException(e); } } private PreparedStatement createAndCacheInsertStatement(StatementType insertType, String key) throws SQLException { String insert = dbFlavor.createInsertStatement(insertType, key); PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(insert); PreparedStatement stmt = cachedStatements.putIfAbsent(insertType, insertStatement); if (stmt == null) { return insertStatement; } return stmt; } private PreparedStatement createAndCacheReadStatement(StatementType readType, String key) throws SQLException { String read = dbFlavor.createReadStatement(readType, key); PreparedStatement readStatement = getShardConnectionByKey(key).prepareStatement(read); PreparedStatement stmt = cachedStatements.putIfAbsent(readType, readStatement); if (stmt == null) { return readStatement; } return stmt; } private PreparedStatement createAndCacheDeleteStatement(StatementType deleteType, String key) throws SQLException { String delete = dbFlavor.createDeleteStatement(deleteType, key); PreparedStatement deleteStatement = getShardConnectionByKey(key).prepareStatement(delete); PreparedStatement stmt = cachedStatements.putIfAbsent(deleteType, deleteStatement); if (stmt == null) { return deleteStatement; } return stmt; } private PreparedStatement createAndCacheUpdateStatement(StatementType updateType, String key) throws SQLException { String update = dbFlavor.createUpdateStatement(updateType, key); PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(update); PreparedStatement stmt = cachedStatements.putIfAbsent(updateType, insertStatement); if (stmt == null) { return insertStatement; } return stmt; } private PreparedStatement createAndCacheScanStatement(StatementType scanType, String key) throws SQLException { String select = dbFlavor.createScanStatement(scanType, key); PreparedStatement scanStatement = getShardConnectionByKey(key).prepareStatement(select); if (this.jdbcFetchSize > 0) { scanStatement.setFetchSize(this.jdbcFetchSize); } PreparedStatement stmt = cachedStatements.putIfAbsent(scanType, scanStatement); if (stmt == null) { return scanStatement; } return stmt; } @Override public Status read(String tableName, String key, Set fields, Map result) { try { StatementType type = new StatementType(StatementType.Type.READ, tableName, 1, "", getShardIndexByKey(key)); PreparedStatement readStatement = cachedStatements.get(type); if (readStatement == null) { readStatement = createAndCacheReadStatement(type, key); } readStatement.setString(1, key); ResultSet resultSet = readStatement.executeQuery(); if (!resultSet.next()) { resultSet.close(); return Status.NOT_FOUND; } if (result != null && fields != null) { for (String field : fields) { String value = resultSet.getString(field); result.put(field, new StringByteIterator(value)); } } resultSet.close(); return Status.OK; } catch (SQLException e) { System.err.println("Error in processing read of table " + tableName + ": " + e); return Status.ERROR; } } @Override public Status scan(String tableName, String startKey, int recordcount, Set fields, Vector> result) { try { StatementType type = new StatementType(StatementType.Type.SCAN, tableName, 1, "", getShardIndexByKey(startKey)); PreparedStatement scanStatement = cachedStatements.get(type); if (scanStatement == null) { scanStatement = createAndCacheScanStatement(type, startKey); } scanStatement.setString(1, startKey); scanStatement.setInt(2, recordcount); ResultSet resultSet = scanStatement.executeQuery(); for (int i = 0; i < recordcount && resultSet.next(); i++) { if (result != null && fields != null) { HashMap values = new HashMap(); for (String field : fields) { String value = resultSet.getString(field); values.put(field, new StringByteIterator(value)); } result.add(values); } } resultSet.close(); return Status.OK; } catch (SQLException e) { System.err.println("Error in processing scan of table: " + tableName + e); return Status.ERROR; } } @Override public Status update(String tableName, String key, Map values) { try { int numFields = values.size(); OrderedFieldInfo fieldInfo = getFieldInfo(values); StatementType type = new StatementType(StatementType.Type.UPDATE, tableName, numFields, fieldInfo.getFieldKeys(), getShardIndexByKey(key)); PreparedStatement updateStatement = cachedStatements.get(type); if (updateStatement == null) { updateStatement = createAndCacheUpdateStatement(type, key); } int index = 1; for (String value: fieldInfo.getFieldValues()) { updateStatement.setString(index++, value); } updateStatement.setString(index, key); int result = updateStatement.executeUpdate(); if (result == 1) { return Status.OK; } return Status.UNEXPECTED_STATE; } catch (SQLException e) { System.err.println("Error in processing update to table: " + tableName + e); return Status.ERROR; } } @Override public Status insert(String tableName, String key, Map values) { try { int numFields = values.size(); OrderedFieldInfo fieldInfo = getFieldInfo(values); StatementType type = new StatementType(StatementType.Type.INSERT, tableName, numFields, fieldInfo.getFieldKeys(), getShardIndexByKey(key)); PreparedStatement insertStatement = cachedStatements.get(type); if (insertStatement == null) { insertStatement = createAndCacheInsertStatement(type, key); } insertStatement.setString(1, key); int index = 2; for (String value: fieldInfo.getFieldValues()) { insertStatement.setString(index++, value); } // Using the batch insert API if (batchUpdates) { insertStatement.addBatch(); // Check for a sane batch size if (batchSize > 0) { // Commit the batch after it grows beyond the configured size if (++numRowsInBatch % batchSize == 0) { int[] results = insertStatement.executeBatch(); for (int r : results) { if (r != 1) { return Status.ERROR; } } // If autoCommit is off, make sure we commit the batch if (!autoCommit) { getShardConnectionByKey(key).commit(); } return Status.OK; } // else, the default value of -1 or a nonsense. Treat it as an infinitely large batch. } // else, we let the batch accumulate // Added element to the batch, potentially committing the batch too. return Status.BATCHED_OK; } else { // Normal update int result = insertStatement.executeUpdate(); // If we are not autoCommit, we might have to commit now if (!autoCommit) { // Let updates be batcher locally if (batchSize > 0) { if (++numRowsInBatch % batchSize == 0) { // Send the batch of updates getShardConnectionByKey(key).commit(); } // uhh return Status.OK; } else { // Commit each update getShardConnectionByKey(key).commit(); } } if (result == 1) { return Status.OK; } } return Status.UNEXPECTED_STATE; } catch (SQLException e) { System.err.println("Error in processing insert to table: " + tableName + e); return Status.ERROR; } } @Override public Status delete(String tableName, String key) { try { StatementType type = new StatementType(StatementType.Type.DELETE, tableName, 1, "", getShardIndexByKey(key)); PreparedStatement deleteStatement = cachedStatements.get(type); if (deleteStatement == null) { deleteStatement = createAndCacheDeleteStatement(type, key); } deleteStatement.setString(1, key); int result = deleteStatement.executeUpdate(); if (result == 1) { return Status.OK; } return Status.UNEXPECTED_STATE; } catch (SQLException e) { System.err.println("Error in processing delete to table: " + tableName + e); return Status.ERROR; } } private OrderedFieldInfo getFieldInfo(Map values) { String fieldKeys = ""; List fieldValues = new ArrayList<>(); int count = 0; for (Map.Entry entry : values.entrySet()) { fieldKeys += entry.getKey(); if (count < values.size() - 1) { fieldKeys += ","; } fieldValues.add(count, entry.getValue().toString()); count++; } return new OrderedFieldInfo(fieldKeys, fieldValues); } } ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBCreateTable.java ================================================ /** * Copyright (c) 2010 - 2016 Yahoo! Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.io.FileInputStream; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Enumeration; import java.util.Properties; /** * Utility class to create the table to be used by the benchmark. * * @author sudipto */ public final class JdbcDBCreateTable { private static void usageMessage() { System.out.println("Create Table Client. Options:"); System.out.println(" -p key=value properties defined."); System.out.println(" -P location of the properties file to load."); System.out.println(" -n name of the table."); System.out.println(" -f number of fields (default 10)."); } private static void createTable(Properties props, String tablename) throws SQLException { String driver = props.getProperty(JdbcDBClient.DRIVER_CLASS); String username = props.getProperty(JdbcDBClient.CONNECTION_USER); String password = props.getProperty(JdbcDBClient.CONNECTION_PASSWD, ""); String url = props.getProperty(JdbcDBClient.CONNECTION_URL); int fieldcount = Integer.parseInt(props.getProperty(JdbcDBClient.FIELD_COUNT_PROPERTY, JdbcDBClient.FIELD_COUNT_PROPERTY_DEFAULT)); if (driver == null || username == null || url == null) { throw new SQLException("Missing connection information."); } Connection conn = null; try { Class.forName(driver); conn = DriverManager.getConnection(url, username, password); Statement stmt = conn.createStatement(); StringBuilder sql = new StringBuilder("DROP TABLE IF EXISTS "); sql.append(tablename); sql.append(";"); stmt.execute(sql.toString()); sql = new StringBuilder("CREATE TABLE "); sql.append(tablename); sql.append(" (YCSB_KEY VARCHAR PRIMARY KEY"); for (int idx = 0; idx < fieldcount; idx++) { sql.append(", FIELD"); sql.append(idx); sql.append(" TEXT"); } sql.append(");"); stmt.execute(sql.toString()); System.out.println("Table " + tablename + " created.."); } catch (ClassNotFoundException e) { throw new SQLException("JDBC Driver class not found."); } finally { if (conn != null) { System.out.println("Closing database connection."); conn.close(); } } } /** * @param args */ public static void main(String[] args) { if (args.length == 0) { usageMessage(); System.exit(0); } String tablename = null; int fieldcount = -1; Properties props = new Properties(); Properties fileprops = new Properties(); // parse arguments int argindex = 0; while (args[argindex].startsWith("-")) { if (args[argindex].compareTo("-P") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } String propfile = args[argindex]; argindex++; Properties myfileprops = new Properties(); try { myfileprops.load(new FileInputStream(propfile)); } catch (IOException e) { System.out.println(e.getMessage()); System.exit(0); } // Issue #5 - remove call to stringPropertyNames to make compilable // under Java 1.5 for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, myfileprops.getProperty(prop)); } } else if (args[argindex].compareTo("-p") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } int eq = args[argindex].indexOf('='); if (eq < 0) { usageMessage(); System.exit(0); } String name = args[argindex].substring(0, eq); String value = args[argindex].substring(eq + 1); props.put(name, value); argindex++; } else if (args[argindex].compareTo("-n") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } tablename = args[argindex++]; } else if (args[argindex].compareTo("-f") == 0) { argindex++; if (argindex >= args.length) { usageMessage(); System.exit(0); } try { fieldcount = Integer.parseInt(args[argindex++]); } catch (NumberFormatException e) { System.err.println("Invalid number for field count"); usageMessage(); System.exit(1); } } else { System.out.println("Unknown option " + args[argindex]); usageMessage(); System.exit(0); } if (argindex >= args.length) { break; } } if (argindex != args.length) { usageMessage(); System.exit(0); } // overwrite file properties with properties from the command line // Issue #5 - remove call to stringPropertyNames to make compilable under // Java 1.5 for (Enumeration e = props.propertyNames(); e.hasMoreElements();) { String prop = (String) e.nextElement(); fileprops.setProperty(prop, props.getProperty(prop)); } props = fileprops; if (tablename == null) { System.err.println("table name missing."); usageMessage(); System.exit(1); } if (fieldcount > 0) { props.setProperty(JdbcDBClient.FIELD_COUNT_PROPERTY, String.valueOf(fieldcount)); } try { createTable(props, tablename); } catch (SQLException e) { System.err.println("Error in creating table. " + e); System.exit(1); } } /** * Hidden constructor. */ private JdbcDBCreateTable() { super(); } } ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/StatementType.java ================================================ /** * Copyright (c) 2010 Yahoo! Inc., 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; /** * The statement type for the prepared statements. */ public class StatementType { enum Type { INSERT(1), DELETE(2), READ(3), UPDATE(4), SCAN(5); private final int internalType; private Type(int type) { internalType = type; } int getHashCode() { final int prime = 31; int result = 1; result = prime * result + internalType; return result; } } private Type type; private int shardIndex; private int numFields; private String tableName; private String fieldString; public StatementType(Type type, String tableName, int numFields, String fieldString, int shardIndex) { this.type = type; this.tableName = tableName; this.numFields = numFields; this.fieldString = fieldString; this.shardIndex = shardIndex; } public String getTableName() { return tableName; } public String getFieldString() { return fieldString; } public int getNumFields() { return numFields; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + numFields + 100 * shardIndex; result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); result = prime * result + ((type == null) ? 0 : type.getHashCode()); return result; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } StatementType other = (StatementType) obj; if (numFields != other.numFields) { return false; } if (shardIndex != other.shardIndex) { return false; } if (tableName == null) { if (other.tableName != null) { return false; } } else if (!tableName.equals(other.tableName)) { return false; } if (type != other.type) { return false; } if (!fieldString.equals(other.fieldString)) { return false; } return true; } } ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/flavors/DBFlavor.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.flavors; import com.yahoo.ycsb.db.StatementType; /** * DBFlavor captures minor differences in syntax and behavior among JDBC implementations and SQL * dialects. This class also acts as a factory to instantiate concrete flavors based on the JDBC URL. */ public abstract class DBFlavor { enum DBName { DEFAULT, PHOENIX } private final DBName dbName; public DBFlavor(DBName dbName) { this.dbName = dbName; } public static DBFlavor fromJdbcUrl(String url) { if (url.startsWith("jdbc:phoenix")) { return new PhoenixDBFlavor(); } return new DefaultDBFlavor(); } /** * Create and return a SQL statement for inserting data. */ public abstract String createInsertStatement(StatementType insertType, String key); /** * Create and return a SQL statement for reading data. */ public abstract String createReadStatement(StatementType readType, String key); /** * Create and return a SQL statement for deleting data. */ public abstract String createDeleteStatement(StatementType deleteType, String key); /** * Create and return a SQL statement for updating data. */ public abstract String createUpdateStatement(StatementType updateType, String key); /** * Create and return a SQL statement for scanning data. */ public abstract String createScanStatement(StatementType scanType, String key); } ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/flavors/DefaultDBFlavor.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.flavors; import com.yahoo.ycsb.db.JdbcDBClient; import com.yahoo.ycsb.db.StatementType; /** * A default flavor for relational databases. */ public class DefaultDBFlavor extends DBFlavor { public DefaultDBFlavor() { super(DBName.DEFAULT); } public DefaultDBFlavor(DBName dbName) { super(dbName); } @Override public String createInsertStatement(StatementType insertType, String key) { StringBuilder insert = new StringBuilder("INSERT INTO "); insert.append(insertType.getTableName()); insert.append(" (" + JdbcDBClient.PRIMARY_KEY + "," + insertType.getFieldString() + ")"); insert.append(" VALUES(?"); for (int i = 0; i < insertType.getNumFields(); i++) { insert.append(",?"); } insert.append(")"); return insert.toString(); } @Override public String createReadStatement(StatementType readType, String key) { StringBuilder read = new StringBuilder("SELECT * FROM "); read.append(readType.getTableName()); read.append(" WHERE "); read.append(JdbcDBClient.PRIMARY_KEY); read.append(" = "); read.append("?"); return read.toString(); } @Override public String createDeleteStatement(StatementType deleteType, String key) { StringBuilder delete = new StringBuilder("DELETE FROM "); delete.append(deleteType.getTableName()); delete.append(" WHERE "); delete.append(JdbcDBClient.PRIMARY_KEY); delete.append(" = ?"); return delete.toString(); } @Override public String createUpdateStatement(StatementType updateType, String key) { String[] fieldKeys = updateType.getFieldString().split(","); StringBuilder update = new StringBuilder("UPDATE "); update.append(updateType.getTableName()); update.append(" SET "); for (int i = 0; i < fieldKeys.length; i++) { update.append(fieldKeys[i]); update.append("=?"); if (i < fieldKeys.length - 1) { update.append(", "); } } update.append(" WHERE "); update.append(JdbcDBClient.PRIMARY_KEY); update.append(" = ?"); return update.toString(); } @Override public String createScanStatement(StatementType scanType, String key) { StringBuilder select = new StringBuilder("SELECT * FROM "); select.append(scanType.getTableName()); select.append(" WHERE "); select.append(JdbcDBClient.PRIMARY_KEY); select.append(" >= ?"); select.append(" ORDER BY "); select.append(JdbcDBClient.PRIMARY_KEY); select.append(" LIMIT ?"); return select.toString(); } } ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/flavors/PhoenixDBFlavor.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.flavors; import com.yahoo.ycsb.db.JdbcDBClient; import com.yahoo.ycsb.db.StatementType; /** * Database flavor for Apache Phoenix. Captures syntax differences used by Phoenix. */ public class PhoenixDBFlavor extends DefaultDBFlavor { public PhoenixDBFlavor() { super(DBName.PHOENIX); } @Override public String createInsertStatement(StatementType insertType, String key) { // Phoenix uses UPSERT syntax StringBuilder insert = new StringBuilder("UPSERT INTO "); insert.append(insertType.getTableName()); insert.append(" (" + JdbcDBClient.PRIMARY_KEY + "," + insertType.getFieldString() + ")"); insert.append(" VALUES(?"); for (int i = 0; i < insertType.getNumFields(); i++) { insert.append(",?"); } insert.append(")"); return insert.toString(); } @Override public String createUpdateStatement(StatementType updateType, String key) { // Phoenix doesn't have UPDATE semantics, just re-use UPSERT VALUES on the specific columns String[] fieldKeys = updateType.getFieldString().split(","); StringBuilder update = new StringBuilder("UPSERT INTO "); update.append(updateType.getTableName()); update.append(" ("); // Each column to update for (int i = 0; i < fieldKeys.length; i++) { update.append(fieldKeys[i]).append(","); } // And then set the primary key column update.append(JdbcDBClient.PRIMARY_KEY).append(") VALUES("); // Add an unbound param for each column to update for (int i = 0; i < fieldKeys.length; i++) { update.append("?, "); } // Then the primary key column's value update.append("?)"); return update.toString(); } } ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/flavors/package-info.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * This package contains a collection of database-specific overrides. This accounts for the variance * that can be present where JDBC does not explicitly define what a database must do or when a * database has a non-standard SQL implementation. */ package com.yahoo.ycsb.db.flavors; ================================================ FILE: jdbc/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014 - 2016, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for stores that can be accessed via JDBC. */ package com.yahoo.ycsb.db; ================================================ FILE: jdbc/src/main/resources/sql/README.md ================================================ Contains all the SQL statements used by the JDBC client. ================================================ FILE: jdbc/src/main/resources/sql/create_table.mysql ================================================ -- Copyright (c) 2015 YCSB contributors. All rights reserved. -- -- Licensed under the Apache License, Version 2.0 (the "License"); you -- may not use this file except in compliance with the License. You -- may obtain a copy of the License at -- -- http://www.apache.org/licenses/LICENSE-2.0 -- -- Unless required by applicable law or agreed to in writing, software -- distributed under the License is distributed on an "AS IS" BASIS, -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -- implied. See the License for the specific language governing -- permissions and limitations under the License. See accompanying -- LICENSE file. -- Creates a Table. -- Drop the table if it exists; DROP TABLE IF EXISTS usertable; -- Create the user table with 5 fields. CREATE TABLE usertable(YCSB_KEY VARCHAR (255) PRIMARY KEY, FIELD0 TEXT, FIELD1 TEXT, FIELD2 TEXT, FIELD3 TEXT, FIELD4 TEXT, FIELD5 TEXT, FIELD6 TEXT, FIELD7 TEXT, FIELD8 TEXT, FIELD9 TEXT); ================================================ FILE: jdbc/src/main/resources/sql/create_table.sql ================================================ -- Copyright (c) 2015 YCSB contributors. All rights reserved. -- -- Licensed under the Apache License, Version 2.0 (the "License"); you -- may not use this file except in compliance with the License. You -- may obtain a copy of the License at -- -- http://www.apache.org/licenses/LICENSE-2.0 -- -- Unless required by applicable law or agreed to in writing, software -- distributed under the License is distributed on an "AS IS" BASIS, -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -- implied. See the License for the specific language governing -- permissions and limitations under the License. See accompanying -- LICENSE file. -- Creates a Table. -- Drop the table if it exists; DROP TABLE IF EXISTS usertable; -- Create the user table with 5 fields. CREATE TABLE usertable(YCSB_KEY VARCHAR PRIMARY KEY, FIELD0 VARCHAR, FIELD1 VARCHAR, FIELD2 VARCHAR, FIELD3 VARCHAR, FIELD4 VARCHAR, FIELD5 VARCHAR, FIELD6 VARCHAR, FIELD7 VARCHAR, FIELD8 VARCHAR, FIELD9 VARCHAR); ================================================ FILE: jdbc/src/test/java/com/yahoo/ycsb/db/JdbcDBClientTest.java ================================================ /** * Copyright (c) 2015 - 2016 Yahoo! Inc., 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static org.junit.Assert.*; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; import org.junit.*; import java.sql.*; import java.util.HashMap; import java.util.Map; import java.util.HashSet; import java.util.Set; import java.util.Properties; import java.util.Vector; public class JdbcDBClientTest { private static final String TEST_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver"; private static final String TEST_DB_URL = "jdbc:hsqldb:mem:ycsb"; private static final String TEST_DB_USER = "sa"; private static final String TABLE_NAME = "USERTABLE"; private static final int FIELD_LENGTH = 32; private static final String FIELD_PREFIX = "FIELD"; private static final String KEY_PREFIX = "user"; private static final String KEY_FIELD = "YCSB_KEY"; private static final int NUM_FIELDS = 3; private static Connection jdbcConnection = null; private static JdbcDBClient jdbcDBClient = null; @BeforeClass public static void setup() { setupWithBatch(1, true); } public static void setupWithBatch(int batchSize, boolean autoCommit) { try { jdbcConnection = DriverManager.getConnection(TEST_DB_URL); jdbcDBClient = new JdbcDBClient(); Properties p = new Properties(); p.setProperty(JdbcDBClient.CONNECTION_URL, TEST_DB_URL); p.setProperty(JdbcDBClient.DRIVER_CLASS, TEST_DB_DRIVER); p.setProperty(JdbcDBClient.CONNECTION_USER, TEST_DB_USER); p.setProperty(JdbcDBClient.DB_BATCH_SIZE, Integer.toString(batchSize)); p.setProperty(JdbcDBClient.JDBC_BATCH_UPDATES, "true"); p.setProperty(JdbcDBClient.JDBC_AUTO_COMMIT, Boolean.toString(autoCommit)); jdbcDBClient.setProperties(p); jdbcDBClient.init(); } catch (SQLException e) { e.printStackTrace(); fail("Could not create local Database"); } catch (DBException e) { e.printStackTrace(); fail("Could not create JdbcDBClient instance"); } } @AfterClass public static void teardown() { try { if (jdbcConnection != null) { jdbcConnection.close(); } } catch (SQLException e) { e.printStackTrace(); } try { if (jdbcDBClient != null) { jdbcDBClient.cleanup(); } } catch (DBException e) { e.printStackTrace(); } } @Before public void prepareTest() { try { DatabaseMetaData metaData = jdbcConnection.getMetaData(); ResultSet tableResults = metaData.getTables(null, null, TABLE_NAME, null); if (tableResults.next()) { // If the table already exists, just truncate it jdbcConnection.prepareStatement( String.format("TRUNCATE TABLE %s", TABLE_NAME) ).execute(); } else { // If the table does not exist then create it StringBuilder createString = new StringBuilder( String.format("CREATE TABLE %s (%s VARCHAR(100) PRIMARY KEY", TABLE_NAME, KEY_FIELD) ); for (int i = 0; i < NUM_FIELDS; i++) { createString.append( String.format(", %s%d VARCHAR(100)", FIELD_PREFIX, i) ); } createString.append(")"); jdbcConnection.prepareStatement(createString.toString()).execute(); } } catch (SQLException e) { e.printStackTrace(); fail("Failed to prepare test"); } } /* This is a copy of buildDeterministicValue() from core:com.yahoo.ycsb.workloads.CoreWorkload.java. That method is neither public nor static so we need a copy. */ private String buildDeterministicValue(String key, String fieldkey) { int size = FIELD_LENGTH; StringBuilder sb = new StringBuilder(size); sb.append(key); sb.append(':'); sb.append(fieldkey); while (sb.length() < size) { sb.append(':'); sb.append(sb.toString().hashCode()); } sb.setLength(size); return sb.toString(); } /* Inserts a row of deterministic values for the given insertKey using the jdbcDBClient. */ private HashMap insertRow(String insertKey) { HashMap insertMap = new HashMap(); for (int i = 0; i < 3; i++) { insertMap.put(FIELD_PREFIX + i, new StringByteIterator(buildDeterministicValue(insertKey, FIELD_PREFIX + i))); } jdbcDBClient.insert(TABLE_NAME, insertKey, insertMap); return insertMap; } @Test public void insertTest() { try { String insertKey = "user0"; HashMap insertMap = insertRow(insertKey); ResultSet resultSet = jdbcConnection.prepareStatement( String.format("SELECT * FROM %s", TABLE_NAME) ).executeQuery(); // Check we have a result Row assertTrue(resultSet.next()); // Check that all the columns have expected values assertEquals(resultSet.getString(KEY_FIELD), insertKey); for (int i = 0; i < 3; i++) { assertEquals(resultSet.getString(FIELD_PREFIX + i), insertMap.get(FIELD_PREFIX + i).toString()); } // Check that we do not have any more rows assertFalse(resultSet.next()); resultSet.close(); } catch (SQLException e) { e.printStackTrace(); fail("Failed insertTest"); } } @Test public void updateTest() { try { String preupdateString = "preupdate"; StringBuilder fauxInsertString = new StringBuilder( String.format("INSERT INTO %s VALUES(?", TABLE_NAME) ); for (int i = 0; i < NUM_FIELDS; i++) { fauxInsertString.append(",?"); } fauxInsertString.append(")"); PreparedStatement fauxInsertStatement = jdbcConnection.prepareStatement(fauxInsertString.toString()); for (int i = 2; i < NUM_FIELDS + 2; i++) { fauxInsertStatement.setString(i, preupdateString); } fauxInsertStatement.setString(1, "user0"); fauxInsertStatement.execute(); fauxInsertStatement.setString(1, "user1"); fauxInsertStatement.execute(); fauxInsertStatement.setString(1, "user2"); fauxInsertStatement.execute(); HashMap updateMap = new HashMap(); for (int i = 0; i < 3; i++) { updateMap.put(FIELD_PREFIX + i, new StringByteIterator(buildDeterministicValue("user1", FIELD_PREFIX + i))); } jdbcDBClient.update(TABLE_NAME, "user1", updateMap); ResultSet resultSet = jdbcConnection.prepareStatement( String.format("SELECT * FROM %s ORDER BY %s", TABLE_NAME, KEY_FIELD) ).executeQuery(); // Ensure that user0 record was not changed resultSet.next(); assertEquals("Assert first row key is user0", resultSet.getString(KEY_FIELD), "user0"); for (int i = 0; i < 3; i++) { assertEquals("Assert first row fields contain preupdateString", resultSet.getString(FIELD_PREFIX + i), preupdateString); } // Check that all the columns have expected values for user1 record resultSet.next(); assertEquals(resultSet.getString(KEY_FIELD), "user1"); for (int i = 0; i < 3; i++) { assertEquals(resultSet.getString(FIELD_PREFIX + i), updateMap.get(FIELD_PREFIX + i).toString()); } // Ensure that user2 record was not changed resultSet.next(); assertEquals("Assert third row key is user2", resultSet.getString(KEY_FIELD), "user2"); for (int i = 0; i < 3; i++) { assertEquals("Assert third row fields contain preupdateString", resultSet.getString(FIELD_PREFIX + i), preupdateString); } resultSet.close(); } catch (SQLException e) { e.printStackTrace(); fail("Failed updateTest"); } } @Test public void readTest() { String insertKey = "user0"; HashMap insertMap = insertRow(insertKey); Set readFields = new HashSet(); HashMap readResultMap = new HashMap(); // Test reading a single field readFields.add("FIELD0"); jdbcDBClient.read(TABLE_NAME, insertKey, readFields, readResultMap); assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size()); for (String field: readFields) { assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString()); } readResultMap = new HashMap(); // Test reading all fields readFields.add("FIELD1"); readFields.add("FIELD2"); jdbcDBClient.read(TABLE_NAME, insertKey, readFields, readResultMap); assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size()); for (String field: readFields) { assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString()); } } @Test public void deleteTest() { try { insertRow("user0"); String deleteKey = "user1"; insertRow(deleteKey); insertRow("user2"); jdbcDBClient.delete(TABLE_NAME, deleteKey); ResultSet resultSet = jdbcConnection.prepareStatement( String.format("SELECT * FROM %s", TABLE_NAME) ).executeQuery(); int totalRows = 0; while (resultSet.next()) { assertNotEquals("Assert this is not the deleted row key", deleteKey, resultSet.getString(KEY_FIELD)); totalRows++; } // Check we do not have a result Row assertEquals("Assert we ended with the correct number of rows", totalRows, 2); resultSet.close(); } catch (SQLException e) { e.printStackTrace(); fail("Failed deleteTest"); } } @Test public void scanTest() throws SQLException { Map> keyMap = new HashMap>(); for (int i = 0; i < 5; i++) { String insertKey = KEY_PREFIX + i; keyMap.put(insertKey, insertRow(insertKey)); } Set fieldSet = new HashSet(); fieldSet.add("FIELD0"); fieldSet.add("FIELD1"); int startIndex = 1; int resultRows = 3; Vector> resultVector = new Vector>(); jdbcDBClient.scan(TABLE_NAME, KEY_PREFIX + startIndex, resultRows, fieldSet, resultVector); // Check the resultVector is the correct size assertEquals("Assert the correct number of results rows were returned", resultRows, resultVector.size()); // Check each vector row to make sure we have the correct fields int testIndex = startIndex; for (Map result: resultVector) { assertEquals("Assert that this row has the correct number of fields", fieldSet.size(), result.size()); for (String field: fieldSet) { assertEquals("Assert this field is correct in this row", keyMap.get(KEY_PREFIX + testIndex).get(field).toString(), result.get(field).toString()); } testIndex++; } } @Test public void insertBatchTest() throws DBException { insertBatchTest(20); } @Test public void insertPartialBatchTest() throws DBException { insertBatchTest(19); } public void insertBatchTest(int numRows) throws DBException { teardown(); setupWithBatch(10, false); try { String insertKey = "user0"; HashMap insertMap = insertRow(insertKey); assertEquals(3, insertMap.size()); ResultSet resultSet = jdbcConnection.prepareStatement( String.format("SELECT * FROM %s", TABLE_NAME) ).executeQuery(); // Check we do not have a result Row (because batch is not full yet) assertFalse(resultSet.next()); // insert more rows, completing 1 batch (still results are partial). for (int i = 1; i < numRows; i++) { insertMap = insertRow("user" + i); } // assertNumRows(10 * (numRows / 10)); // call cleanup, which should insert the partial batch jdbcDBClient.cleanup(); // Prevent a teardown() from printing an error jdbcDBClient = null; // Check that we have all rows assertNumRows(numRows); } catch (SQLException e) { e.printStackTrace(); fail("Failed insertBatchTest"); } finally { teardown(); // for next tests setup(); } } private void assertNumRows(long numRows) throws SQLException { ResultSet resultSet = jdbcConnection.prepareStatement( String.format("SELECT * FROM %s", TABLE_NAME) ).executeQuery(); for (int i = 0; i < numRows; i++) { assertTrue("expecting " + numRows + " results, received only " + i, resultSet.next()); } assertFalse("expecting " + numRows + " results, received more", resultSet.next()); resultSet.close(); } } ================================================ FILE: kudu/README.md ================================================ # Kudu bindings for YCSB [Apache Kudu](https://kudu.apache.org) is a storage engine that enables fast analytics on fast data. ## Benchmarking Kudu Use the following command line to load the initial data into an existing Kudu cluster with default configurations. ``` bin/ycsb load kudu -P workloads/workloada ``` Additional configurations: * `kudu_master_addresses`: The master's address. The default configuration expects a master on localhost. * `kudu_pre_split_num_tablets`: The number of tablets (or partitions) to create for the table. The default uses 4 tablets. A good rule of thumb is to use 5 per tablet server. * `kudu_table_num_replicas`: The number of replicas that each tablet will have. The default is 3. Should only be configured to use 1 instead, for single node tests. * `kudu_sync_ops`: If the client should wait after every write operation. The default is true. * `kudu_block_size`: The data block size used to configure columns. The default is 4096 bytes. Then, you can run the workload: ``` bin/ycsb run kudu -P workloads/workloada ``` ## Using a previous client version If you wish to use a different Kudu client version than the one shipped with YCSB, you can specify on the command line with `-Dkudu.version=x`. For example: ``` mvn -pl com.yahoo.ycsb:kudu-binding -am package -DskipTests -Dkudu.version=1.0.1 ``` Note that only versions since 1.0 are supported, since Kudu did not guarantee wire or API compatibility prior to 1.0. ================================================ FILE: kudu/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent kudu-binding Kudu DB Binding jar org.apache.kudu kudu-client ${kudu.version} com.yahoo.ycsb core ${project.version} provided org.slf4j slf4j-api 1.7.21 org.slf4j slf4j-log4j12 1.7.21 ================================================ FILE: kudu/src/main/conf/log4j.properties ================================================ # # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Enables getting logs from the client. log4j.rootLogger = INFO, out log4j.appender.out = org.apache.log4j.ConsoleAppender log4j.appender.out.layout = org.apache.log4j.PatternLayout log4j.appender.out.layout.ConversionPattern = %d (%t) [%p - %l] %m%n log4j.logger.kudu = INFO ================================================ FILE: kudu/src/main/java/com/yahoo/ycsb/db/KuduYCSBClient.java ================================================ /** * Copyright (c) 2015-2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.stumbleupon.async.TimeoutException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.workloads.CoreWorkload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.*; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.Vector; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; import static org.apache.kudu.Type.STRING; import static org.apache.kudu.client.KuduPredicate.ComparisonOp.EQUAL; import static org.apache.kudu.client.KuduPredicate.ComparisonOp.GREATER_EQUAL; /** * Kudu client for YCSB framework. Example to load:

* * 
 * 
 * $ ./bin/ycsb load kudu -P workloads/workloada -threads 5
 * 
 *
* *
Example to run:
* * 
 * 
 * ./bin/ycsb run kudu -P workloads/workloada -p kudu_sync_ops=true -threads 5
 * 
 *
* *
*/ public class KuduYCSBClient extends com.yahoo.ycsb.DB { private static final Logger LOG = LoggerFactory.getLogger(KuduYCSBClient.class); private static final String KEY = "key"; private static final Status TIMEOUT = new Status("TIMEOUT", "The operation timed out."); private static final int MAX_TABLETS = 9000; private static final long DEFAULT_SLEEP = 60000; private static final String SYNC_OPS_OPT = "kudu_sync_ops"; private static final String PRE_SPLIT_NUM_TABLETS_OPT = "kudu_pre_split_num_tablets"; private static final String TABLE_NUM_REPLICAS = "kudu_table_num_replicas"; private static final String BLOCK_SIZE_OPT = "kudu_block_size"; private static final String MASTER_ADDRESSES_OPT = "kudu_master_addresses"; private static final int BLOCK_SIZE_DEFAULT = 4096; private static final List COLUMN_NAMES = new ArrayList<>(); private static KuduClient client; private static Schema schema; private KuduSession session; private KuduTable kuduTable; @Override public void init() throws DBException { String tableName = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); initClient(tableName, getProperties()); this.session = client.newSession(); if (getProperties().getProperty(SYNC_OPS_OPT) != null && getProperties().getProperty(SYNC_OPS_OPT).equals("false")) { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); this.session.setMutationBufferSpace(100); } else { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_SYNC); } try { this.kuduTable = client.openTable(tableName); } catch (Exception e) { throw new DBException("Could not open a table because of:", e); } } private static synchronized void initClient(String tableName, Properties prop) throws DBException { if (client != null) { return; } String masterAddresses = prop.getProperty(MASTER_ADDRESSES_OPT); if (masterAddresses == null) { masterAddresses = "localhost:7051"; } int numTablets = getIntFromProp(prop, PRE_SPLIT_NUM_TABLETS_OPT, 4); if (numTablets > MAX_TABLETS) { throw new DBException(String.format( "Specified number of tablets (%s) must be equal or below %s", numTablets, MAX_TABLETS)); } int numReplicas = getIntFromProp(prop, TABLE_NUM_REPLICAS, 3); int blockSize = getIntFromProp(prop, BLOCK_SIZE_OPT, BLOCK_SIZE_DEFAULT); client = new KuduClient.KuduClientBuilder(masterAddresses) .defaultSocketReadTimeoutMs(DEFAULT_SLEEP) .defaultOperationTimeoutMs(DEFAULT_SLEEP) .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP) .build(); LOG.debug("Connecting to the masters at {}", masterAddresses); int fieldCount = getIntFromProp(prop, CoreWorkload.FIELD_COUNT_PROPERTY, Integer.parseInt(CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT)); List columns = new ArrayList<>(fieldCount + 1); ColumnSchema keyColumn = new ColumnSchema.ColumnSchemaBuilder(KEY, STRING) .key(true) .desiredBlockSize(blockSize) .build(); columns.add(keyColumn); COLUMN_NAMES.add(KEY); for (int i = 0; i < fieldCount; i++) { String name = "field" + i; COLUMN_NAMES.add(name); columns.add(new ColumnSchema.ColumnSchemaBuilder(name, STRING) .desiredBlockSize(blockSize) .build()); } schema = new Schema(columns); CreateTableOptions builder = new CreateTableOptions(); builder.setRangePartitionColumns(new ArrayList()); List hashPartitionColumns = new ArrayList<>(); hashPartitionColumns.add(KEY); builder.addHashPartitions(hashPartitionColumns, numTablets); builder.setNumReplicas(numReplicas); try { client.createTable(tableName, schema, builder); } catch (Exception e) { if (!e.getMessage().contains("already exists")) { throw new DBException("Couldn't create the table", e); } } } private static int getIntFromProp(Properties prop, String propName, int defaultValue) throws DBException { String intStr = prop.getProperty(propName); if (intStr == null) { return defaultValue; } else { try { return Integer.valueOf(intStr); } catch (NumberFormatException ex) { throw new DBException("Provided number for " + propName + " isn't a valid integer"); } } } @Override public void cleanup() throws DBException { try { this.session.close(); } catch (Exception e) { throw new DBException("Couldn't cleanup the session", e); } } @Override public Status read(String table, String key, Set fields, Map result) { Vector> results = new Vector<>(); final Status status = scan(table, key, 1, fields, results); if (!status.equals(Status.OK)) { return status; } if (results.size() != 1) { return Status.NOT_FOUND; } result.putAll(results.firstElement()); return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { KuduScanner.KuduScannerBuilder scannerBuilder = client.newScannerBuilder(kuduTable); List querySchema; if (fields == null) { querySchema = COLUMN_NAMES; // No need to set the projected columns with the whole schema. } else { querySchema = new ArrayList<>(fields); scannerBuilder.setProjectedColumnNames(querySchema); } ColumnSchema column = schema.getColumnByIndex(0); KuduPredicate.ComparisonOp predicateOp = recordcount == 1 ? EQUAL : GREATER_EQUAL; KuduPredicate predicate = KuduPredicate.newComparisonPredicate(column, predicateOp, startkey); scannerBuilder.addPredicate(predicate); scannerBuilder.limit(recordcount); // currently noop KuduScanner scanner = scannerBuilder.build(); while (scanner.hasMoreRows()) { RowResultIterator data = scanner.nextRows(); addAllRowsToResult(data, recordcount, querySchema, result); if (recordcount == result.size()) { break; } } RowResultIterator closer = scanner.close(); addAllRowsToResult(closer, recordcount, querySchema, result); } catch (TimeoutException te) { LOG.info("Waited too long for a scan operation with start key={}", startkey); return TIMEOUT; } catch (Exception e) { LOG.warn("Unexpected exception", e); return Status.ERROR; } return Status.OK; } private void addAllRowsToResult(RowResultIterator it, int recordcount, List querySchema, Vector> result) throws Exception { RowResult row; HashMap rowResult = new HashMap<>(querySchema.size()); if (it == null) { return; } while (it.hasNext()) { if (result.size() == recordcount) { return; } row = it.next(); int colIdx = 0; for (String col : querySchema) { rowResult.put(col, new StringByteIterator(row.getString(colIdx))); colIdx++; } result.add(rowResult); } } @Override public Status update(String table, String key, Map values) { Update update = this.kuduTable.newUpdate(); PartialRow row = update.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { String columnName = schema.getColumnByIndex(i).getName(); if (values.containsKey(columnName)) { String value = values.get(columnName).toString(); row.addString(columnName, value); } } apply(update); return Status.OK; } @Override public Status insert(String table, String key, Map values) { Insert insert = this.kuduTable.newInsert(); PartialRow row = insert.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { row.addString(i, values.get(schema.getColumnByIndex(i).getName()).toString()); } apply(insert); return Status.OK; } @Override public Status delete(String table, String key) { Delete delete = this.kuduTable.newDelete(); PartialRow row = delete.getRow(); row.addString(KEY, key); apply(delete); return Status.OK; } private void apply(Operation op) { try { OperationResponse response = session.apply(op); if (response != null && response.hasRowError()) { LOG.info("Write operation failed: {}", response.getRowError()); } } catch (KuduException ex) { LOG.warn("Write operation failed", ex); } } } ================================================ FILE: kudu/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2015-2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Apache Kudu. */ package com.yahoo.ycsb.db; ================================================ FILE: kudu/src/main/resources/log4j.properties ================================================ # Root logger option log4j.rootLogger=DEBUG, stderr log4j.logger.com.stumbleupon.async=WARN log4j.logger.org.apache.kudu=WARN # Direct log messages to stderr log4j.appender.stderr = org.apache.log4j.ConsoleAppender log4j.appender.stderr.Target=System.err log4j.appender.stderr.layout = org.apache.log4j.PatternLayout log4j.appender.stderr.layout.ConversionPattern = [%p] %d{HH:mm:ss.SSS} (%F:%L) %m%n ================================================ FILE: leveldb/README.md ================================================ Interface to leveldb for integration into ycsb. Based on simpleleveldb, a http daemon for leveldb. Make sure to start the daemon before running any workloads. To open shell for interactive debugging: ./bin/ycsb shell leveldb leveldb: https://code.google.com/p/leveldb/ simpleleveldb: https://github.com/bitly/simplehttp/tree/master/simpleleveldb ================================================ FILE: leveldb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb root 0.1.4 leveldb-binding LevelDB Binding commons-httpclient commons-httpclient 3.1 org.apache.httpcomponents httpclient 4.2.6 org.mongodb mongo-java-driver ${mongodb.version} com.googlecode.json-simple json-simple 1.1 com.yahoo.ycsb core ${project.version} org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} jar-with-dependencies false package single ================================================ FILE: leveldb/src/main/java/com/yahoo/ycsb/db/LevelDbClient.java ================================================ /** * LevelDB client binding for YCSB. */ package com.yahoo.ycsb.db; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URLEncoder; import java.text.MessageFormat; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import com.mongodb.BasicDBObject; import com.mongodb.DBObject; import com.mongodb.util.JSON; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; /** * LevelDB client for YCSB framework. */ public class LevelDbClient extends DB { private static String dbUrl = "http://localhost:8080"; private static String deleteUrl = dbUrl + "/del"; private static String insertUrl = dbUrl + "/put"; private static String readUrl = dbUrl + "/get"; private static String scanUrl = dbUrl + "/fwmatch"; private static JSONParser parser = new JSONParser(); private static DefaultHttpClient httpClient; private static HttpPost httpPost; private static HttpResponse response; private static HttpGet httpGet; // having multiple tables in leveldb is a hack. must divide key // space into logical tables private static Map tableKeyPrefix; private static final AtomicInteger prefix = new AtomicInteger(0); private static String getStringFromInputStream(InputStream is) { BufferedReader br = null; StringBuilder sb = new StringBuilder(); String line; try { br = new BufferedReader(new InputStreamReader(is)); while ((line = br.readLine()) != null) { sb.append(line); } } catch (IOException e) { e.printStackTrace(); } finally { if (br != null) { try { br.close(); } catch (IOException e) { e.printStackTrace(); } } } return sb.toString(); } /** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. */ @Override public void init() throws DBException { httpClient = new DefaultHttpClient(); tableKeyPrefix = new HashMap(); } /** * Cleanup any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void cleanup() throws DBException { } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int delete(String table, String key) { try { httpPost = new HttpPost(MessageFormat.format("{0}?key={1}", deleteUrl, key)); // System.out.println("# delete request - " + // httpPost.getRequestLine()); response = httpClient.execute(httpPost); // System.out.println("# delete response - " + // getStringFromInputStream(response.getEntity().getContent())); EntityUtils.consume(response.getEntity()); return response.getStatusLine().getStatusCode() == 200 ? 0 : 1; } catch (Exception e) { System.err.println(e.toString()); return 1; } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int insert(String table, String key, HashMap values) { try { JSONObject jsonValues = new JSONObject(); for (Entry entry : StringByteIterator.getStringMap( values).entrySet()) { jsonValues.put(entry.getKey(), entry.getValue()); } String urlStringValues = URLEncoder.encode( jsonValues.toJSONString(), "UTF-8"); httpPost = new HttpPost(MessageFormat.format( "{0}?key={1}&value={2}", insertUrl, key, urlStringValues)); // System.out.println("# insert request - " + // httpPost.getRequestLine()); response = httpClient.execute(httpPost); // System.out.println("# insert response - " + // getStringFromInputStream(response.getEntity().getContent())); EntityUtils.consume(response.getEntity()); return response.getStatusLine().getStatusCode() == 200 ? 0 : 1; } catch (Exception e) { e.printStackTrace(); return 1; } } /** * Read a record from the database. Each field/value pair from the result * will be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public int read(String table, String key, Set fields, HashMap result) { try { httpGet = new HttpGet(MessageFormat.format("{0}?key={1}", readUrl, key)); // System.out.println("# read request - " + // httpGet.getRequestLine()); response = httpClient.execute(httpGet); JSONObject jsonResponse = (JSONObject) parser.parse(EntityUtils .toString(response.getEntity())); // System.out.println("# read response - " + jsonResponse); // Use Mongo DBObject to encode back to ByteIterator DBObject bson = (DBObject) JSON.parse(jsonResponse.get("data") .toString()); if (bson != null) { result.putAll(bson.toMap()); } EntityUtils.consume(response.getEntity()); return bson != null ? 0 : 1; } catch (Exception e) { System.err.println(e.toString()); return 1; } } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int update(String table, String key, HashMap values) { try { httpGet = new HttpGet(MessageFormat.format("{0}?key={1}", readUrl, key)); // System.out.println("# update read request - " + // httpGet.getRequestLine()); response = httpClient.execute(httpGet); JSONObject jsonResponse = (JSONObject) parser.parse(EntityUtils .toString(response.getEntity())); // System.out.println("# update read response - " + jsonResponse); JSONObject existingValues = new JSONObject(); // check if key exists in the db if (response.getStatusLine().getStatusCode() == 200) { existingValues = (JSONObject) parser.parse(jsonResponse.get( "data").toString()); } for (Entry entry : StringByteIterator.getStringMap( values).entrySet()) { existingValues.put(entry.getKey(), entry.getValue()); } String urlStringValues = URLEncoder.encode( existingValues.toJSONString(), "UTF-8"); httpPost = new HttpPost(MessageFormat.format( "{0}?key={1}&value={2}", insertUrl, key, urlStringValues)); // System.out.println("# update insert request - " + // httpPost.getRequestLine()); response = httpClient.execute(httpPost); // System.out.println("# update insert response - " + // getStringFromInputStream(response.getEntity().getContent())); EntityUtils.consume(response.getEntity()); return response.getStatusLine().getStatusCode() == 200 ? 0 : 1; } catch (Exception e) { System.err.println(e.toString()); return 1; } } /** * Perform a range scan for a set of records in the database. Each * field/value pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { httpGet = new HttpGet(MessageFormat.format("{0}?key={1}&limit={2}", scanUrl, startkey, recordcount)); // System.out.println("# scan request - " + // httpGet.getRequestLine()); response = httpClient.execute(httpGet); JSONObject jsonResponse = (JSONObject) parser.parse(EntityUtils .toString(response.getEntity())); // System.out.println("# scan response - " + jsonResponse); JSONArray scanEntries = (JSONArray) parser.parse(jsonResponse.get( "data").toString()); for (Object e : scanEntries) { JSONObject entry = (JSONObject) e; DBObject value = (DBObject) JSON.parse(entry.get("value") .toString()); DBObject bsonResult = new BasicDBObject(); if (fields == null) { bsonResult = value; } else { for (String s : fields) { // get has same result for missing keys and values that // are null bsonResult.put(s, value.get(s)); } } HashMap singleResult = new HashMap(); singleResult.putAll(bsonResult.toMap()); result.addElement(singleResult); } EntityUtils.consume(response.getEntity()); return response.getStatusLine().getStatusCode() == 200 ? 0 : 1; } catch (Exception e) { System.err.println(e.toString()); return 1; } } } ================================================ FILE: leveldbjni/README.md ================================================ ## Quick Start - The way to connect ycsb and your own leveldb This section describes how to run YCSB on Redis. ### 1. Code JNI for leveldb and get a leveldb-jni.jar ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:redis-binding -am clean package ### 4. Set the dependcy about leveldb-jni.jar in pom.xml org.fusesource.leveldbjni leveldbjni-all 1.7 system ${project.basedir}/lib/leveldbjni-all-1.7.jar ### 5. Load data and run tests Load the data: ./bin/ycsb load redis -s -P workloads/workloada > outputLoad.txt Run the workload test: ./bin/ycsb run redis -s -P workloads/workloada > outputRun.txt ================================================ FILE: leveldbjni/pom.xml ================================================ 4.0.0 com.yahoo.ycsb root 0.1.4 leveldbjni-binding LevelDBJNI Binding com.yahoo.ycsb core ${project.version} org.fusesource.leveldbjni leveldbjni-all 1.7 system ${project.basedir}/lib/leveldbjni-all-1.7.jar org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} jar-with-dependencies false package single ================================================ FILE: leveldbjni/src/main/java/com/yahoo/ycsb/db/LevelDbJniClient.java ================================================ /** * LevelDB client binding for YCSB. */ package com.yahoo.ycsb.db; import static org.fusesource.leveldbjni.JniDBFactory.factory; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; import org.iq80.leveldb.DBIterator; import org.iq80.leveldb.Options; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; /** * LevelDBJni client for YCSB framework. */ public class LevelDbJniClient extends DB { private static org.iq80.leveldb.DB db = null; private static final AtomicInteger initCount = new AtomicInteger(0); private synchronized static void getDBInstance() { if (db == null) { Options options = new Options(); // options.cacheSize(100 * 1048576); // 100MB cache options.createIfMissing(true); try { db = factory.open(new File("leveldb_database"), options); } catch (IOException e) { System.out.println("Failed to open database"); e.printStackTrace(); } } } private static byte[] mapToBytes(Map map) throws IOException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(byteOut); out.writeObject(map); return byteOut.toByteArray(); } private static Map bytesToMap(byte[] bytes) throws IOException, ClassNotFoundException { ByteArrayInputStream byteIn = new ByteArrayInputStream(bytes); ObjectInputStream in = new ObjectInputStream(byteIn); @SuppressWarnings("unchecked") Map map = (Map) in.readObject(); return map; } /** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. */ @Override public void init() throws DBException { initCount.incrementAndGet(); getDBInstance(); } /** * Cleanup any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void cleanup() throws DBException { if (initCount.decrementAndGet() <= 0) { try { db.close(); } catch (IOException e) { System.out.println("Failed to close db"); e.printStackTrace(); } } } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int delete(String table, String key) { db.delete(key.getBytes()); return 0; } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int insert(String table, String key, HashMap values) { Map stringValues = StringByteIterator .getStringMap(values); try { db.put(key.getBytes(), mapToBytes(stringValues)); } catch (org.iq80.leveldb.DBException e) { System.out.println("Failed to insert " + key); e.printStackTrace(); return 1; } catch (IOException e) { System.out.println("Failed to insert " + key); e.printStackTrace(); return 1; } return 0; } /** * Read a record from the database. Each field/value pair from the result * will be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public int read(String table, String key, Set fields, HashMap result) { byte[] value = db.get(key.getBytes()); if (value == null) { return 1; } Map map; try { map = bytesToMap(value); } catch (IOException e) { System.out.println("Failed to read " + key); e.printStackTrace(); return 1; } catch (ClassNotFoundException e) { System.out.println("Failed to read " + key); e.printStackTrace(); return 1; } StringByteIterator.putAllAsByteIterators(result, map); return 0; } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int update(String table, String key, HashMap values) { byte[] existingBytes = db.get(key.getBytes()); Map existingValues; if (existingBytes != null) { try { existingValues = bytesToMap(existingBytes); } catch (IOException e) { System.out.println("Failed to read for update " + key); e.printStackTrace(); return 1; } catch (ClassNotFoundException e) { System.out.println("Failed to read for update " + key); e.printStackTrace(); return 1; } } else { existingValues = new HashMap(); } Map newValues = StringByteIterator.getStringMap(values); existingValues.putAll(newValues); try { db.put(key.getBytes(), mapToBytes(existingValues)); } catch (org.iq80.leveldb.DBException e) { System.out.println("Failed to insert " + key); e.printStackTrace(); return 1; } catch (IOException e) { System.out.println("Failed to insert " + key); e.printStackTrace(); return 1; } return 0; } /** * Perform a range scan for a set of records in the database. Each * field/value pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { DBIterator iterator = db.iterator(); int count = 0; try { iterator.seek(startkey.getBytes()); while (iterator.hasNext() && count < recordcount) { String key = iterator.peekNext().getKey().toString(); if (fields == null || fields.contains(key)) { HashMap value; value = (HashMap) bytesToMap(iterator .peekNext().getValue()); HashMap byteValues = new HashMap(); StringByteIterator.putAllAsByteIterators(byteValues, value); result.addElement(byteValues); } iterator.next(); count += 1; } } catch (IOException e) { System.out.println("Failed to scan"); e.printStackTrace(); return 1; } catch (ClassNotFoundException e) { System.out.println("Failed to scan"); e.printStackTrace(); return 1; } finally { try { iterator.close(); } catch (IOException e) { System.out.println("Failed to close iterator"); e.printStackTrace(); } } return 0; } } ================================================ FILE: mapkeeper/README.md ================================================ # MapKeeper-Specific Properties ## mapkeeper.host Specifies the host MapKeeper server is running on (default: localhost). ## mapkeeper.port Specifies the port MapKeeper server is listening to (default: 9090). ================================================ FILE: mapkeeper/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent mapkeeper-binding Mapkeeper DB Binding jar com.yahoo.mapkeeper mapkeeper ${mapkeeper.version} com.yahoo.ycsb core ${project.version} provided mapkeeper-releases https://raw.github.com/m1ch1/m1ch1-mvn-repo/master/releases ================================================ FILE: mapkeeper/src/main/java/com/yahoo/ycsb/db/MapKeeperClient.java ================================================ /** * Copyright (c) 2012 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import com.yahoo.mapkeeper.BinaryResponse; import com.yahoo.mapkeeper.MapKeeper; import com.yahoo.mapkeeper.Record; import com.yahoo.mapkeeper.RecordListResponse; import com.yahoo.mapkeeper.ResponseCode; import com.yahoo.mapkeeper.ScanOrder; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.StringByteIterator; import com.yahoo.ycsb.workloads.CoreWorkload; public class MapKeeperClient extends DB { private static final String HOST = "mapkeeper.host"; private static final String HOST_DEFAULT = "localhost"; private static final String PORT = "mapkeeper.port"; private static final String PORT_DEFAULT = "9090"; MapKeeper.Client c; boolean writeallfields; static boolean initteddb = false; private synchronized static void initDB(Properties p, MapKeeper.Client c) throws TException { if(!initteddb) { initteddb = true; c.addMap(p.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT)); } } public void init() { String host = getProperties().getProperty(HOST, HOST_DEFAULT); int port = Integer.parseInt(getProperties().getProperty(PORT, PORT_DEFAULT)); TTransport tr = new TFramedTransport(new TSocket(host, port)); TProtocol proto = new TBinaryProtocol(tr); c = new MapKeeper.Client(proto); try { tr.open(); initDB(getProperties(), c); } catch(TException e) { throw new RuntimeException(e); } writeallfields = Boolean.parseBoolean(getProperties().getProperty(CoreWorkload.WRITE_ALL_FIELDS_PROPERTY, CoreWorkload.WRITE_ALL_FIELDS_PROPERTY_DEFAULT)); } ByteBuffer encode(HashMap values) { int len = 0; for(Map.Entry entry : values.entrySet()) { len += (entry.getKey().length() + 1 + entry.getValue().bytesLeft() + 1); } byte[] array = new byte[len]; int i = 0; for(Map.Entry entry : values.entrySet()) { for(int j = 0; j < entry.getKey().length(); j++) { array[i] = (byte)entry.getKey().charAt(j); i++; } array[i] = '\t'; // XXX would like to use sane delimiter (null, 254, 255, ...) but java makes this nearly impossible i++; ByteIterator v = entry.getValue(); i = v.nextBuf(array, i); array[i] = '\t'; i++; } array[array.length-1] = 0; ByteBuffer buf = ByteBuffer.wrap(array); buf.rewind(); return buf; } void decode(Set fields, String tups, HashMap tup) { String[] tok = tups.split("\\t"); if(tok.length == 0) { throw new IllegalStateException("split returned empty array!"); } for(int i = 0; i < tok.length; i+=2) { if(fields == null || fields.contains(tok[i])) { if(tok.length < i+2) { throw new IllegalStateException("Couldn't parse tuple <" + tups + "> at index " + i); } if(tok[i] == null || tok[i+1] == null) throw new NullPointerException("Key is " + tok[i] + " val is + " + tok[i+1]); tup.put(tok[i], new StringByteIterator(tok[i+1])); } } if(tok.length == 0) { System.err.println("Empty tuple: " + tups); } } int ycsbThriftRet(BinaryResponse succ, ResponseCode zero, ResponseCode one) { return ycsbThriftRet(succ.responseCode, zero, one); } int ycsbThriftRet(ResponseCode rc, ResponseCode zero, ResponseCode one) { return rc == zero ? 0 : rc == one ? 1 : 2; } ByteBuffer bufStr(String str) { ByteBuffer buf = ByteBuffer.wrap(str.getBytes()); return buf; } String strResponse(BinaryResponse buf) { return new String(buf.value.array()); } @Override public int read(String table, String key, Set fields, Map result) { try { ByteBuffer buf = bufStr(key); BinaryResponse succ = c.get(table, buf); int ret = ycsbThriftRet( succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound); if(ret == 0) { decode(fields, strResponse(succ), result); } return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { //XXX what to pass in for nulls / zeros? RecordListResponse res = c.scan(table, ScanOrder.Ascending, bufStr(startkey), true, null, false, recordcount, 0); int ret = ycsbThriftRet(res.responseCode, ResponseCode.Success, ResponseCode.ScanEnded); if(ret == 0) { for(Record r : res.records) { HashMap tuple = new HashMap(); // Note: r.getKey() and r.getValue() call special helper methods that trim the buffer // to an appropriate length, and memcpy it to a byte[]. Trying to manipulate the ByteBuffer // directly leads to trouble. tuple.put("key", new StringByteIterator(new String(r.getKey()))); decode(fields, new String(r.getValue())/*strBuf(r.bufferForValue())*/, tuple); result.add(tuple); } } return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int update(String table, String key, Map values) { try { if(!writeallfields) { HashMap oldval = new HashMap(); read(table, key, null, oldval); for(Map.Entry entry : values.entrySet()) { oldval.put(entry.getKey(), entry.getValue())); } values = oldval; } ResponseCode succ = c.update(table, bufStr(key), encode(values)); return ycsbThriftRet(succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound); } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int insert(String table, String key, Map values) { try { int ret = ycsbThriftRet(c.insert(table, bufStr(key), encode(values)), ResponseCode.Success, ResponseCode.RecordExists); return ret; } catch(TException e) { e.printStackTrace(); return 2; } } @Override public int delete(String table, String key) { try { return ycsbThriftRet(c.remove(table, bufStr(key)), ResponseCode.Success, ResponseCode.RecordExists); } catch(TException e) { e.printStackTrace(); return 2; } } } ================================================ FILE: memcached/README.md ================================================ # YCSB Memcached binding This section describes how to run YCSB on memcached. ## 1. Install and start memcached service on the host(s) Debian / Ubuntu: sudo apt-get install memcached RedHat / CentOS: sudo yum install memcached ## 2. Install Java and Maven See step 2 in [`../mongodb/README.md`](../mongodb/README.md). ## 3. Set up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:memcached-binding -am clean package ## 4. Load data and run tests Load the data: ./bin/ycsb load memcached -s -P workloads/workloada > outputLoad.txt Run the workload test: ./bin/ycsb run memcached -s -P workloads/workloada > outputRun.txt ## 5. memcached Connection Parameters A sample configuration is provided in [`conf/memcached.properties`](conf/memcached.properties). ### Required params - `memcached.hosts` This is a comma-separated list of hosts providing the memcached interface. You can use IPs or hostnames. The port is optional and defaults to the memcached standard port of `11211` if not specified. ### Optional params - `memcached.shutdownTimeoutMillis` Shutdown timeout in milliseconds. - `memcached.objectExpirationTime` Object expiration time for memcached; defaults to `Integer.MAX_VALUE`. - `memcached.checkOperationStatus` Whether to verify the success of each operation; defaults to true. - `memcached.readBufferSize` Read buffer size, in bytes. - `memcached.opTimeoutMillis` Operation timeout, in milliseconds. - `memcached.failureMode` What to do with failures; this is one of `net.spy.memcached.FailureMode` enum values, which are currently: `Redistribute`, `Retry`, or `Cancel`. - `memcached.protocol` Set to 'binary' to use memcached binary protocol. Set to 'text' or omit this field to use memcached text protocol You can set properties on the command line via `-p`, e.g.: ./bin/ycsb load memcached -s -P workloads/workloada \ -p "memcached.hosts=127.0.0.1" > outputLoad.txt ================================================ FILE: memcached/conf/memcached.properties ================================================ # Copyright (c) 2015 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Sample property file for Memcached Client ## Mandatory parameters # A comma-separated list of memcached server endpoints, each being an IP or # hostname with an optional port; the port defaults to the memcached-standard # port of 11211 if not specified. # # memcached.hosts = ## Optional parameters # Shutdown timeout in milliseconds. # # memcached.shutdownTimeoutMillis = 30000 # Object expiration time for memcached; defaults to `Integer.MAX_VALUE`. # # memcached.objectExpirationTime = 2147483647 # Whether to verify the success of each operation; defaults to true. # # memcached.checkOperationStatus = true # Read buffer size, in bytes. # # memcached.readBufferSize = 3000000 # Operation timeout, in milliseconds. # # memcached.opTimeoutMillis = 60000 # What to do with failures; this is one of `net.spy.memcached.FailureMode` enum # values, which are currently: `Redistribute`, `Retry`, or `Cancel`. # # memcached.failureMode = Redistribute ================================================ FILE: memcached/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent memcached-binding memcached binding jar log4j log4j 1.2.17 com.yahoo.ycsb core ${project.version} org.codehaus.jackson jackson-mapper-asl 1.9.13 net.spy spymemcached 2.11.4 org.apache.maven.plugins maven-assembly-plugin ${maven.assembly.version} jar-with-dependencies false package single ================================================ FILE: memcached/src/main/java/com/yahoo/ycsb/db/MemcachedClient.java ================================================ /** * Copyright (c) 2014-2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.net.InetSocketAddress; import java.text.MessageFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.Vector; import net.spy.memcached.ConnectionFactoryBuilder; import net.spy.memcached.FailureMode; // We also use `net.spy.memcached.MemcachedClient`; it is not imported // explicitly and referred to with its full path to avoid conflicts with the // class of the same name in this file. import net.spy.memcached.internal.GetFuture; import net.spy.memcached.internal.OperationFuture; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.node.ObjectNode; import org.apache.log4j.Logger; import static java.util.concurrent.TimeUnit.MILLISECONDS; /** * Concrete Memcached client implementation. */ public class MemcachedClient extends DB { private final Logger logger = Logger.getLogger(getClass()); protected static final ObjectMapper MAPPER = new ObjectMapper(); private boolean checkOperationStatus; private long shutdownTimeoutMillis; private int objectExpirationTime; public static final String HOSTS_PROPERTY = "memcached.hosts"; public static final int DEFAULT_PORT = 11211; private static final String TEMPORARY_FAILURE_MSG = "Temporary failure"; private static final String CANCELLED_MSG = "cancelled"; public static final String SHUTDOWN_TIMEOUT_MILLIS_PROPERTY = "memcached.shutdownTimeoutMillis"; public static final String DEFAULT_SHUTDOWN_TIMEOUT_MILLIS = "30000"; public static final String OBJECT_EXPIRATION_TIME_PROPERTY = "memcached.objectExpirationTime"; public static final String DEFAULT_OBJECT_EXPIRATION_TIME = String.valueOf(Integer.MAX_VALUE); public static final String CHECK_OPERATION_STATUS_PROPERTY = "memcached.checkOperationStatus"; public static final String CHECK_OPERATION_STATUS_DEFAULT = "true"; public static final String READ_BUFFER_SIZE_PROPERTY = "memcached.readBufferSize"; public static final String DEFAULT_READ_BUFFER_SIZE = "3000000"; public static final String OP_TIMEOUT_PROPERTY = "memcached.opTimeoutMillis"; public static final String DEFAULT_OP_TIMEOUT = "60000"; public static final String FAILURE_MODE_PROPERTY = "memcached.failureMode"; public static final FailureMode FAILURE_MODE_PROPERTY_DEFAULT = FailureMode.Redistribute; public static final String PROTOCOL_PROPERTY = "memcached.protocol"; public static final ConnectionFactoryBuilder.Protocol DEFAULT_PROTOCOL = ConnectionFactoryBuilder.Protocol.TEXT; /** * The MemcachedClient implementation that will be used to communicate * with the memcached server. */ private net.spy.memcached.MemcachedClient client; /** * @returns Underlying Memcached protocol client, implemented by * SpyMemcached. */ protected net.spy.memcached.MemcachedClient memcachedClient() { return client; } @Override public void init() throws DBException { try { client = createMemcachedClient(); checkOperationStatus = Boolean.parseBoolean( getProperties().getProperty(CHECK_OPERATION_STATUS_PROPERTY, CHECK_OPERATION_STATUS_DEFAULT)); objectExpirationTime = Integer.parseInt( getProperties().getProperty(OBJECT_EXPIRATION_TIME_PROPERTY, DEFAULT_OBJECT_EXPIRATION_TIME)); shutdownTimeoutMillis = Integer.parseInt( getProperties().getProperty(SHUTDOWN_TIMEOUT_MILLIS_PROPERTY, DEFAULT_SHUTDOWN_TIMEOUT_MILLIS)); } catch (Exception e) { throw new DBException(e); } } protected net.spy.memcached.MemcachedClient createMemcachedClient() throws Exception { ConnectionFactoryBuilder connectionFactoryBuilder = new ConnectionFactoryBuilder(); connectionFactoryBuilder.setReadBufferSize(Integer.parseInt( getProperties().getProperty(READ_BUFFER_SIZE_PROPERTY, DEFAULT_READ_BUFFER_SIZE))); connectionFactoryBuilder.setOpTimeout(Integer.parseInt( getProperties().getProperty(OP_TIMEOUT_PROPERTY, DEFAULT_OP_TIMEOUT))); String protocolString = getProperties().getProperty(PROTOCOL_PROPERTY); connectionFactoryBuilder.setProtocol( protocolString == null ? DEFAULT_PROTOCOL : ConnectionFactoryBuilder.Protocol.valueOf(protocolString.toUpperCase())); String failureString = getProperties().getProperty(FAILURE_MODE_PROPERTY); connectionFactoryBuilder.setFailureMode( failureString == null ? FAILURE_MODE_PROPERTY_DEFAULT : FailureMode.valueOf(failureString)); // Note: this only works with IPv4 addresses due to its assumption of // ":" being the separator of hostname/IP and port; this is not the case // when dealing with IPv6 addresses. // // TODO(mbrukman): fix this. List addresses = new ArrayList(); String[] hosts = getProperties().getProperty(HOSTS_PROPERTY).split(","); for (String address : hosts) { int colon = address.indexOf(":"); int port = DEFAULT_PORT; String host = address; if (colon != -1) { port = Integer.parseInt(address.substring(colon + 1)); host = address.substring(0, colon); } addresses.add(new InetSocketAddress(host, port)); } return new net.spy.memcached.MemcachedClient( connectionFactoryBuilder.build(), addresses); } @Override public Status read( String table, String key, Set fields, Map result) { key = createQualifiedKey(table, key); try { GetFuture future = memcachedClient().asyncGet(key); Object document = future.get(); if (document != null) { fromJson((String) document, fields, result); } return Status.OK; } catch (Exception e) { logger.error("Error encountered for key: " + key, e); return Status.ERROR; } } @Override public Status scan( String table, String startkey, int recordcount, Set fields, Vector> result){ return Status.NOT_IMPLEMENTED; } @Override public Status update( String table, String key, Map values) { key = createQualifiedKey(table, key); try { OperationFuture future = memcachedClient().replace(key, objectExpirationTime, toJson(values)); return getReturnCode(future); } catch (Exception e) { logger.error("Error updating value with key: " + key, e); return Status.ERROR; } } @Override public Status insert( String table, String key, Map values) { key = createQualifiedKey(table, key); try { OperationFuture future = memcachedClient().add(key, objectExpirationTime, toJson(values)); return getReturnCode(future); } catch (Exception e) { logger.error("Error inserting value", e); return Status.ERROR; } } @Override public Status delete(String table, String key) { key = createQualifiedKey(table, key); try { OperationFuture future = memcachedClient().delete(key); return getReturnCode(future); } catch (Exception e) { logger.error("Error deleting value", e); return Status.ERROR; } } protected Status getReturnCode(OperationFuture future) { if (!checkOperationStatus) { return Status.OK; } if (future.getStatus().isSuccess()) { return Status.OK; } else if (TEMPORARY_FAILURE_MSG.equals(future.getStatus().getMessage())) { return new Status("TEMPORARY_FAILURE", TEMPORARY_FAILURE_MSG); } else if (CANCELLED_MSG.equals(future.getStatus().getMessage())) { return new Status("CANCELLED_MSG", CANCELLED_MSG); } return new Status("ERROR", future.getStatus().getMessage()); } @Override public void cleanup() throws DBException { if (client != null) { memcachedClient().shutdown(shutdownTimeoutMillis, MILLISECONDS); } } protected static String createQualifiedKey(String table, String key) { return MessageFormat.format("{0}-{1}", table, key); } protected static void fromJson( String value, Set fields, Map result) throws IOException { JsonNode json = MAPPER.readTree(value); boolean checkFields = fields != null && !fields.isEmpty(); for (Iterator> jsonFields = json.getFields(); jsonFields.hasNext(); /* increment in loop body */) { Map.Entry jsonField = jsonFields.next(); String name = jsonField.getKey(); if (checkFields && fields.contains(name)) { continue; } JsonNode jsonValue = jsonField.getValue(); if (jsonValue != null && !jsonValue.isNull()) { result.put(name, new StringByteIterator(jsonValue.asText())); } } } protected static String toJson(Map values) throws IOException { ObjectNode node = MAPPER.createObjectNode(); Map stringMap = StringByteIterator.getStringMap(values); for (Map.Entry pair : stringMap.entrySet()) { node.put(pair.getKey(), pair.getValue()); } JsonFactory jsonFactory = new JsonFactory(); Writer writer = new StringWriter(); JsonGenerator jsonGenerator = jsonFactory.createJsonGenerator(writer); MAPPER.writeTree(jsonGenerator, node); return writer.toString(); } } ================================================ FILE: memcached/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for memcached. */ package com.yahoo.ycsb.db; ================================================ FILE: mongodb/README.md ================================================ ## Quick Start This section describes how to run YCSB on MongoDB. ### 1. Start MongoDB First, download MongoDB and start `mongod`. For example, to start MongoDB on x86-64 Linux box: wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-x.x.x.tgz tar xfvz mongodb-linux-x86_64-*.tgz mkdir /tmp/mongodb cd mongodb-linux-x86_64-* ./bin/mongod --dbpath /tmp/mongodb Replace x.x.x above with the latest stable release version for MongoDB. See http://docs.mongodb.org/manual/installation/ for installation steps for various operating systems. ### 2. Install Java and Maven Go to http://www.oracle.com/technetwork/java/javase/downloads/index.html and get the url to download the rpm into your server. For example: wget http://download.oracle.com/otn-pub/java/jdk/7u40-b43/jdk-7u40-linux-x64.rpm?AuthParam=11232426132 -o jdk-7u40-linux-x64.rpm rpm -Uvh jdk-7u40-linux-x64.rpm Or install via yum/apt-get sudo yum install java-devel Download MVN from http://maven.apache.org/download.cgi wget http://ftp.heanet.ie/mirrors/www.apache.org/dist/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz sudo tar xzf apache-maven-*-bin.tar.gz -C /usr/local cd /usr/local sudo ln -s apache-maven-* maven sudo vi /etc/profile.d/maven.sh Add the following to `maven.sh` export M2_HOME=/usr/local/maven export PATH=${M2_HOME}/bin:${PATH} Reload bash and test mvn bash mvn -version ### 3. Set Up YCSB Download the YCSB zip file and compile: curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.5.0/ycsb-0.5.0.tar.gz tar xfvz ycsb-0.5.0.tar.gz cd ycsb-0.5.0 ### 4. Run YCSB Now you are ready to run! First, use the asynchronous driver to load the data: ./bin/ycsb load mongodb-async -s -P workloads/workloada > outputLoad.txt Then, run the workload: ./bin/ycsb run mongodb-async -s -P workloads/workloada > outputRun.txt Similarly, to use the synchronous driver from MongoDB Inc. we load the data: ./bin/ycsb load mongodb -s -P workloads/workloada > outputLoad.txt Then, run the workload: ./bin/ycsb run mongodb -s -P workloads/workloada > outputRun.txt See the next section for the list of configuration parameters for MongoDB. ## Log Level Control Due to the mongodb driver defaulting to a log level of DEBUG, a logback.xml file is included with this module that restricts the org.mongodb logging to WARN. You can control this by overriding the logback.xml and defining it in your ycsb command by adding this flag: ``` bin/ycsb run mongodb -jvm-args="-Dlogback.configurationFile=/path/to/logback.xml" ``` ## MongoDB Configuration Parameters - `mongodb.url` - This should be a MongoDB URI or connection string. - See http://docs.mongodb.org/manual/reference/connection-string/ for the standard options. - For the complete set of options for the asynchronous driver see: - http://www.allanbank.com/mongodb-async-driver/apidocs/index.html?com/allanbank/mongodb/MongoDbUri.html - For the complete set of options for the synchronous driver see: - http://api.mongodb.org/java/current/index.html?com/mongodb/MongoClientURI.html - Default value is `mongodb://localhost:27017/ycsb?w=1` - Default value of database is `ycsb` - `mongodb.batchsize` - Useful for the insert workload as it will submit the inserts in batches inproving throughput. - Default value is `1`. - `mongodb.upsert` - Determines if the insert operation performs an update with the upsert operation or a insert. Upserts have the advantage that they will continue to work for a partially loaded data set. - Setting to `true` uses updates, `false` uses insert operations. - Default value is `false`. - `mongodb.writeConcern` - **Deprecated** - Use the `w` and `journal` options on the MongoDB URI provided by the `mongodb.url`. - Allowed values are : - `errors_ignored` - `unacknowledged` - `acknowledged` - `journaled` - `replica_acknowledged` - `majority` - Default value is `acknowledged`. - `mongodb.readPreference` - **Deprecated** - Use the `readPreference` options on the MongoDB URI provided by the `mongodb.url`. - Allowed values are : - `primary` - `primary_preferred` - `secondary` - `secondary_preferred` - `nearest` - Default value is `primary`. - `mongodb.maxconnections` - **Deprecated** - Use the `maxPoolSize` options on the MongoDB URI provided by the `mongodb.url`. - Default value is `100`. - `mongodb.threadsAllowedToBlockForConnectionMultiplier` - **Deprecated** - Use the `waitQueueMultiple` options on the MongoDB URI provided by the `mongodb.url`. - Default value is `5`. For example: ./bin/ycsb load mongodb-async -s -P workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb?w=0 To run with the synchronous driver from MongoDB Inc.: ./bin/ycsb load mongodb -s -P workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb?w=0 ================================================ FILE: mongodb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent mongodb-binding MongoDB Binding jar org.mongodb mongo-java-driver ${mongodb.version} com.allanbank mongodb-async-driver ${mongodb.async.version} com.yahoo.ycsb core ${project.version} provided ch.qos.logback logback-classic 1.1.2 runtime junit junit 4.12 test true always warn false never fail allanbank Allanbank Releases http://www.allanbank.com/repo/ default ================================================ FILE: mongodb/src/main/java/com/yahoo/ycsb/db/AsyncMongoDbClient.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static com.allanbank.mongodb.builder.QueryBuilder.where; import com.allanbank.mongodb.Durability; import com.allanbank.mongodb.LockType; import com.allanbank.mongodb.MongoClient; import com.allanbank.mongodb.MongoClientConfiguration; import com.allanbank.mongodb.MongoCollection; import com.allanbank.mongodb.MongoDatabase; import com.allanbank.mongodb.MongoDbUri; import com.allanbank.mongodb.MongoFactory; import com.allanbank.mongodb.MongoIterator; import com.allanbank.mongodb.ReadPreference; import com.allanbank.mongodb.bson.Document; import com.allanbank.mongodb.bson.Element; import com.allanbank.mongodb.bson.ElementType; import com.allanbank.mongodb.bson.builder.BuilderFactory; import com.allanbank.mongodb.bson.builder.DocumentBuilder; import com.allanbank.mongodb.bson.element.BinaryElement; import com.allanbank.mongodb.builder.BatchedWrite; import com.allanbank.mongodb.builder.BatchedWriteMode; import com.allanbank.mongodb.builder.Find; import com.allanbank.mongodb.builder.Sort; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; /** * MongoDB asynchronous client for YCSB framework using the Asynchronous Java * Driver *

* See the README.md for configuration information. *

* * @author rjm * @see Asynchronous * Java Driver */ public class AsyncMongoDbClient extends DB { /** Used to include a field in a response. */ protected static final int INCLUDE = 1; /** The database to use. */ private static String databaseName; /** Thread local document builder. */ private static final ThreadLocal DOCUMENT_BUILDER = new ThreadLocal() { @Override protected DocumentBuilder initialValue() { return BuilderFactory.start(); } }; /** The write concern for the requests. */ private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); /** The connection to MongoDB. */ private static MongoClient mongoClient; /** The write concern for the requests. */ private static Durability writeConcern; /** Which servers to use for reads. */ private static ReadPreference readPreference; /** The database to MongoDB. */ private MongoDatabase database; /** The batch size to use for inserts. */ private static int batchSize; /** If true then use updates with the upsert option for inserts. */ private static boolean useUpsert; /** The bulk inserts pending for the thread. */ private final BatchedWrite.Builder batchedWrite = BatchedWrite.builder() .mode(BatchedWriteMode.REORDERED); /** The number of writes in the batchedWrite. */ private int batchedWriteCount = 0; /** * Cleanup any state for this DB. Called once per DB instance; there is one DB * instance per client thread. */ @Override public final void cleanup() throws DBException { if (INIT_COUNT.decrementAndGet() == 0) { try { mongoClient.close(); } catch (final Exception e1) { System.err.println("Could not close MongoDB connection pool: " + e1.toString()); e1.printStackTrace(); return; } finally { mongoClient = null; database = null; } } } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public final Status delete(final String table, final String key) { try { final MongoCollection collection = database.getCollection(table); final Document q = BuilderFactory.start().add("_id", key).build(); final long res = collection.delete(q, writeConcern); if (res == 0) { System.err.println("Nothing deleted for key " + key); return Status.NOT_FOUND; } return Status.OK; } catch (final Exception e) { System.err.println(e.toString()); return Status.ERROR; } } /** * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public final void init() throws DBException { final int count = INIT_COUNT.incrementAndGet(); synchronized (AsyncMongoDbClient.class) { final Properties props = getProperties(); if (mongoClient != null) { database = mongoClient.getDatabase(databaseName); // If there are more threads (count) than connections then the // Low latency spin lock is not really needed as we will keep // the connections occupied. if (count > mongoClient.getConfig().getMaxConnectionCount()) { mongoClient.getConfig().setLockType(LockType.MUTEX); } return; } // Set insert batchsize, default 1 - to be YCSB-original equivalent batchSize = Integer.parseInt(props.getProperty("mongodb.batchsize", "1")); // Set is inserts are done as upserts. Defaults to false. useUpsert = Boolean.parseBoolean( props.getProperty("mongodb.upsert", "false")); // Just use the standard connection format URL // http://docs.mongodb.org/manual/reference/connection-string/ // to configure the client. String url = props .getProperty("mongodb.url", "mongodb://localhost:27017/ycsb?w=1"); if (!url.startsWith("mongodb://")) { System.err.println("ERROR: Invalid URL: '" + url + "'. Must be of the form " + "'mongodb://:,:/database?" + "options'. See " + "http://docs.mongodb.org/manual/reference/connection-string/."); System.exit(1); } MongoDbUri uri = new MongoDbUri(url); try { databaseName = uri.getDatabase(); if ((databaseName == null) || databaseName.isEmpty()) { // Default database is "ycsb" if database is not // specified in URL databaseName = "ycsb"; } mongoClient = MongoFactory.createClient(uri); MongoClientConfiguration config = mongoClient.getConfig(); if (!url.toLowerCase().contains("locktype=")) { config.setLockType(LockType.LOW_LATENCY_SPIN); // assumed... } readPreference = config.getDefaultReadPreference(); writeConcern = config.getDefaultDurability(); database = mongoClient.getDatabase(databaseName); System.out.println("mongo connection created with " + url); } catch (final Exception e1) { System.err .println("Could not initialize MongoDB connection pool for Loader: " + e1.toString()); e1.printStackTrace(); return; } } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See the {@link DB} * class's description for a discussion of error codes. */ @Override public final Status insert(final String table, final String key, final Map values) { try { final MongoCollection collection = database.getCollection(table); final DocumentBuilder toInsert = DOCUMENT_BUILDER.get().reset().add("_id", key); final Document query = toInsert.build(); for (final Map.Entry entry : values.entrySet()) { toInsert.add(entry.getKey(), entry.getValue().toArray()); } // Do an upsert. if (batchSize <= 1) { long result; if (useUpsert) { result = collection.update(query, toInsert, /* multi= */false, /* upsert= */true, writeConcern); } else { // Return is not stable pre-SERVER-4381. No exception is success. collection.insert(writeConcern, toInsert); result = 1; } return result == 1 ? Status.OK : Status.NOT_FOUND; } // Use a bulk insert. try { if (useUpsert) { batchedWrite.update(query, toInsert, /* multi= */false, /* upsert= */true); } else { batchedWrite.insert(toInsert); } batchedWriteCount += 1; if (batchedWriteCount < batchSize) { return Status.BATCHED_OK; } long count = collection.write(batchedWrite); if (count == batchedWriteCount) { batchedWrite.reset().mode(BatchedWriteMode.REORDERED); batchedWriteCount = 0; return Status.OK; } System.err.println("Number of inserted documents doesn't match the " + "number sent, " + count + " inserted, sent " + batchedWriteCount); batchedWrite.reset().mode(BatchedWriteMode.REORDERED); batchedWriteCount = 0; return Status.ERROR; } catch (Exception e) { System.err.println("Exception while trying bulk insert with " + batchedWriteCount); e.printStackTrace(); return Status.ERROR; } } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } } /** * Read a record from the database. Each field/value pair from the result will * be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public final Status read(final String table, final String key, final Set fields, final Map result) { try { final MongoCollection collection = database.getCollection(table); final DocumentBuilder query = DOCUMENT_BUILDER.get().reset().add("_id", key); Document queryResult = null; if (fields != null) { final DocumentBuilder fieldsToReturn = BuilderFactory.start(); final Iterator iter = fields.iterator(); while (iter.hasNext()) { fieldsToReturn.add(iter.next(), 1); } final Find.Builder fb = new Find.Builder(query); fb.projection(fieldsToReturn); fb.setLimit(1); fb.setBatchSize(1); fb.readPreference(readPreference); final MongoIterator ci = collection.find(fb.build()); if (ci.hasNext()) { queryResult = ci.next(); ci.close(); } } else { queryResult = collection.findOne(query); } if (queryResult != null) { fillMap(result, queryResult); } return queryResult != null ? Status.OK : Status.NOT_FOUND; } catch (final Exception e) { System.err.println(e.toString()); return Status.ERROR; } } /** * Perform a range scan for a set of records in the database. Each field/value * pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See the {@link DB} * class's description for a discussion of error codes. */ @Override public final Status scan(final String table, final String startkey, final int recordcount, final Set fields, final Vector> result) { try { final MongoCollection collection = database.getCollection(table); final Find.Builder find = Find.builder().query(where("_id").greaterThanOrEqualTo(startkey)) .limit(recordcount).batchSize(recordcount).sort(Sort.asc("_id")) .readPreference(readPreference); if (fields != null) { final DocumentBuilder fieldsDoc = BuilderFactory.start(); for (final String field : fields) { fieldsDoc.add(field, INCLUDE); } find.projection(fieldsDoc); } result.ensureCapacity(recordcount); final MongoIterator cursor = collection.find(find); if (!cursor.hasNext()) { System.err.println("Nothing found in scan for key " + startkey); return Status.NOT_FOUND; } while (cursor.hasNext()) { // toMap() returns a Map but result.add() expects a // Map. Hence, the suppress warnings. final Document doc = cursor.next(); final HashMap docAsMap = new HashMap(); fillMap(docAsMap, doc); result.add(docAsMap); } return Status.OK; } catch (final Exception e) { System.err.println(e.toString()); return Status.ERROR; } } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See the {@link DB} * class's description for a discussion of error codes. */ @Override public final Status update(final String table, final String key, final Map values) { try { final MongoCollection collection = database.getCollection(table); final DocumentBuilder query = BuilderFactory.start().add("_id", key); final DocumentBuilder update = BuilderFactory.start(); final DocumentBuilder fieldsToSet = update.push("$set"); for (final Map.Entry entry : values.entrySet()) { fieldsToSet.add(entry.getKey(), entry.getValue().toArray()); } final long res = collection.update(query, update, false, false, writeConcern); return writeConcern == Durability.NONE || res == 1 ? Status.OK : Status.NOT_FOUND; } catch (final Exception e) { System.err.println(e.toString()); return Status.ERROR; } } /** * Fills the map with the ByteIterators from the document. * * @param result * The map to fill. * @param queryResult * The document to fill from. */ protected final void fillMap(final Map result, final Document queryResult) { for (final Element be : queryResult) { if (be.getType() == ElementType.BINARY) { result.put(be.getName(), new BinaryByteArrayIterator((BinaryElement) be)); } } } /** * BinaryByteArrayIterator provides an adapter from a {@link BinaryElement} to * a {@link ByteIterator}. */ private static final class BinaryByteArrayIterator extends ByteIterator { /** The binary data. */ private final BinaryElement binaryElement; /** The current offset into the binary element. */ private int offset; /** * Creates a new BinaryByteArrayIterator. * * @param element * The {@link BinaryElement} to iterate over. */ public BinaryByteArrayIterator(final BinaryElement element) { this.binaryElement = element; this.offset = 0; } /** * {@inheritDoc} *

* Overridden to return the number of bytes remaining in the iterator. *

*/ @Override public long bytesLeft() { return Math.max(0, binaryElement.length() - offset); } /** * {@inheritDoc} *

* Overridden to return true if there is more data in the * {@link BinaryElement}. *

*/ @Override public boolean hasNext() { return (offset < binaryElement.length()); } /** * {@inheritDoc} *

* Overridden to return the next value and advance the iterator. *

*/ @Override public byte nextByte() { final byte value = binaryElement.get(offset); offset += 1; return value; } } } ================================================ FILE: mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java ================================================ /** * Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /* * MongoDB client binding for YCSB. * * Submitted by Yen Pai on 5/11/2010. * * https://gist.github.com/000a66b8db2caf42467b#file_mongo_database.java */ package com.yahoo.ycsb.db; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import com.mongodb.ReadPreference; import com.mongodb.WriteConcern; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.InsertManyOptions; import com.mongodb.client.model.UpdateOneModel; import com.mongodb.client.model.UpdateOptions; import com.mongodb.client.result.DeleteResult; import com.mongodb.client.result.UpdateResult; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import org.bson.Document; import org.bson.types.Binary; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; /** * MongoDB binding for YCSB framework using the MongoDB Inc. driver *

* See the README.md for configuration information. *

* * @author ypai * @see MongoDB Inc. * driver */ public class MongoDbClient extends DB { /** Used to include a field in a response. */ private static final Integer INCLUDE = Integer.valueOf(1); /** The options to use for inserting many documents. */ private static final InsertManyOptions INSERT_UNORDERED = new InsertManyOptions().ordered(false); /** The options to use for inserting a single document. */ private static final UpdateOptions UPDATE_WITH_UPSERT = new UpdateOptions() .upsert(true); /** * The database name to access. */ private static String databaseName; /** The database name to access. */ private static MongoDatabase database; /** * Count the number of times initialized to teardown on the last * {@link #cleanup()}. */ private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); /** A singleton Mongo instance. */ private static MongoClient mongoClient; /** The default read preference for the test. */ private static ReadPreference readPreference; /** The default write concern for the test. */ private static WriteConcern writeConcern; /** The batch size to use for inserts. */ private static int batchSize; /** If true then use updates with the upsert option for inserts. */ private static boolean useUpsert; /** The bulk inserts pending for the thread. */ private final List bulkInserts = new ArrayList(); /** * Cleanup any state for this DB. Called once per DB instance; there is one DB * instance per client thread. */ @Override public void cleanup() throws DBException { if (INIT_COUNT.decrementAndGet() == 0) { try { mongoClient.close(); } catch (Exception e1) { System.err.println("Could not close MongoDB connection pool: " + e1.toString()); e1.printStackTrace(); return; } finally { database = null; mongoClient = null; } } } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See the {@link DB} * class's description for a discussion of error codes. */ @Override public Status delete(String table, String key) { try { MongoCollection collection = database.getCollection(table); Document query = new Document("_id", key); DeleteResult result = collection.withWriteConcern(writeConcern).deleteOne(query); if (result.wasAcknowledged() && result.getDeletedCount() == 0) { System.err.println("Nothing deleted for key " + key); return Status.NOT_FOUND; } return Status.OK; } catch (Exception e) { System.err.println(e.toString()); return Status.ERROR; } } /** * Initialize any state for this DB. Called once per DB instance; there is one * DB instance per client thread. */ @Override public void init() throws DBException { INIT_COUNT.incrementAndGet(); synchronized (INCLUDE) { if (mongoClient != null) { return; } Properties props = getProperties(); // Set insert batchsize, default 1 - to be YCSB-original equivalent batchSize = Integer.parseInt(props.getProperty("batchsize", "1")); // Set is inserts are done as upserts. Defaults to false. useUpsert = Boolean.parseBoolean( props.getProperty("mongodb.upsert", "false")); // Just use the standard connection format URL // http://docs.mongodb.org/manual/reference/connection-string/ // to configure the client. String url = props.getProperty("mongodb.url", null); boolean defaultedUrl = false; if (url == null) { defaultedUrl = true; url = "mongodb://localhost:27017/ycsb?w=1"; } url = OptionsSupport.updateUrl(url, props); if (!url.startsWith("mongodb://")) { System.err.println("ERROR: Invalid URL: '" + url + "'. Must be of the form " + "'mongodb://:,:/database?options'. " + "http://docs.mongodb.org/manual/reference/connection-string/"); System.exit(1); } try { MongoClientURI uri = new MongoClientURI(url); String uriDb = uri.getDatabase(); if (!defaultedUrl && (uriDb != null) && !uriDb.isEmpty() && !"admin".equals(uriDb)) { databaseName = uriDb; } else { // If no database is specified in URI, use "ycsb" databaseName = "ycsb"; } readPreference = uri.getOptions().getReadPreference(); writeConcern = uri.getOptions().getWriteConcern(); mongoClient = new MongoClient(uri); database = mongoClient.getDatabase(databaseName) .withReadPreference(readPreference) .withWriteConcern(writeConcern); System.out.println("mongo client connection created with " + url); } catch (Exception e1) { System.err .println("Could not initialize MongoDB connection pool for Loader: " + e1.toString()); e1.printStackTrace(); return; } } } /** * Insert a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See the {@link DB} * class's description for a discussion of error codes. */ @Override public Status insert(String table, String key, Map values) { try { MongoCollection collection = database.getCollection(table); Document toInsert = new Document("_id", key); for (Map.Entry entry : values.entrySet()) { toInsert.put(entry.getKey(), entry.getValue().toArray()); } if (batchSize == 1) { if (useUpsert) { // this is effectively an insert, but using an upsert instead due // to current inability of the framework to clean up after itself // between test runs. collection.replaceOne(new Document("_id", toInsert.get("_id")), toInsert, UPDATE_WITH_UPSERT); } else { collection.insertOne(toInsert); } } else { bulkInserts.add(toInsert); if (bulkInserts.size() == batchSize) { if (useUpsert) { List> updates = new ArrayList>(bulkInserts.size()); for (Document doc : bulkInserts) { updates.add(new UpdateOneModel( new Document("_id", doc.get("_id")), doc, UPDATE_WITH_UPSERT)); } collection.bulkWrite(updates); } else { collection.insertMany(bulkInserts, INSERT_UNORDERED); } bulkInserts.clear(); } else { return Status.BATCHED_OK; } } return Status.OK; } catch (Exception e) { System.err.println("Exception while trying bulk insert with " + bulkInserts.size()); e.printStackTrace(); return Status.ERROR; } } /** * Read a record from the database. Each field/value pair from the result will * be stored in a HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public Status read(String table, String key, Set fields, Map result) { try { MongoCollection collection = database.getCollection(table); Document query = new Document("_id", key); FindIterable findIterable = collection.find(query); if (fields != null) { Document projection = new Document(); for (String field : fields) { projection.put(field, INCLUDE); } findIterable.projection(projection); } Document queryResult = findIterable.first(); if (queryResult != null) { fillMap(result, queryResult); } return queryResult != null ? Status.OK : Status.NOT_FOUND; } catch (Exception e) { System.err.println(e.toString()); return Status.ERROR; } } /** * Perform a range scan for a set of records in the database. Each field/value * pair from the result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one record * @return Zero on success, a non-zero error code on error. See the {@link DB} * class's description for a discussion of error codes. */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { MongoCursor cursor = null; try { MongoCollection collection = database.getCollection(table); Document scanRange = new Document("$gte", startkey); Document query = new Document("_id", scanRange); Document sort = new Document("_id", INCLUDE); FindIterable findIterable = collection.find(query).sort(sort).limit(recordcount); if (fields != null) { Document projection = new Document(); for (String fieldName : fields) { projection.put(fieldName, INCLUDE); } findIterable.projection(projection); } cursor = findIterable.iterator(); if (!cursor.hasNext()) { System.err.println("Nothing found in scan for key " + startkey); return Status.ERROR; } result.ensureCapacity(recordcount); while (cursor.hasNext()) { HashMap resultMap = new HashMap(); Document obj = cursor.next(); fillMap(resultMap, obj); result.add(resultMap); } return Status.OK; } catch (Exception e) { System.err.println(e.toString()); return Status.ERROR; } finally { if (cursor != null) { cursor.close(); } } } /** * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's * description for a discussion of error codes. */ @Override public Status update(String table, String key, Map values) { try { MongoCollection collection = database.getCollection(table); Document query = new Document("_id", key); Document fieldsToSet = new Document(); for (Map.Entry entry : values.entrySet()) { fieldsToSet.put(entry.getKey(), entry.getValue().toArray()); } Document update = new Document("$set", fieldsToSet); UpdateResult result = collection.updateOne(query, update); if (result.wasAcknowledged() && result.getMatchedCount() == 0) { System.err.println("Nothing updated for key " + key); return Status.NOT_FOUND; } return Status.OK; } catch (Exception e) { System.err.println(e.toString()); return Status.ERROR; } } /** * Fills the map with the values from the DBObject. * * @param resultMap * The map to fill/ * @param obj * The object to copy values from. */ protected void fillMap(Map resultMap, Document obj) { for (Map.Entry entry : obj.entrySet()) { if (entry.getValue() instanceof Binary) { resultMap.put(entry.getKey(), new ByteArrayByteIterator(((Binary) entry.getValue()).getData())); } } } } ================================================ FILE: mongodb/src/main/java/com/yahoo/ycsb/db/OptionsSupport.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.util.Properties; /** * OptionsSupport provides methods for handling legacy options. * * @author rjm */ public final class OptionsSupport { /** Value for an unavailable property. */ private static final String UNAVAILABLE = "n/a"; /** * Updates the URL with the appropriate attributes if legacy properties are * set and the URL does not have the property already set. * * @param url * The URL to update. * @param props * The legacy properties. * @return The updated URL. */ public static String updateUrl(String url, Properties props) { String result = url; // max connections. final String maxConnections = props.getProperty("mongodb.maxconnections", UNAVAILABLE).toLowerCase(); if (!UNAVAILABLE.equals(maxConnections)) { result = addUrlOption(result, "maxPoolSize", maxConnections); } // Blocked thread multiplier. final String threadsAllowedToBlockForConnectionMultiplier = props .getProperty( "mongodb.threadsAllowedToBlockForConnectionMultiplier", UNAVAILABLE).toLowerCase(); if (!UNAVAILABLE.equals(threadsAllowedToBlockForConnectionMultiplier)) { result = addUrlOption(result, "waitQueueMultiple", threadsAllowedToBlockForConnectionMultiplier); } // write concern String writeConcernType = props.getProperty("mongodb.writeConcern", UNAVAILABLE).toLowerCase(); if (!UNAVAILABLE.equals(writeConcernType)) { if ("errors_ignored".equals(writeConcernType)) { result = addUrlOption(result, "w", "0"); } else if ("unacknowledged".equals(writeConcernType)) { result = addUrlOption(result, "w", "0"); } else if ("acknowledged".equals(writeConcernType)) { result = addUrlOption(result, "w", "1"); } else if ("journaled".equals(writeConcernType)) { result = addUrlOption(result, "journal", "true"); // this is the // documented option // name result = addUrlOption(result, "j", "true"); // but keep this until // MongoDB Java driver // supports "journal" option } else if ("replica_acknowledged".equals(writeConcernType)) { result = addUrlOption(result, "w", "2"); } else if ("majority".equals(writeConcernType)) { result = addUrlOption(result, "w", "majority"); } else { System.err.println("WARNING: Invalid writeConcern: '" + writeConcernType + "' will be ignored. " + "Must be one of [ unacknowledged | acknowledged | " + "journaled | replica_acknowledged | majority ]"); } } // read preference String readPreferenceType = props.getProperty("mongodb.readPreference", UNAVAILABLE).toLowerCase(); if (!UNAVAILABLE.equals(readPreferenceType)) { if ("primary".equals(readPreferenceType)) { result = addUrlOption(result, "readPreference", "primary"); } else if ("primary_preferred".equals(readPreferenceType)) { result = addUrlOption(result, "readPreference", "primaryPreferred"); } else if ("secondary".equals(readPreferenceType)) { result = addUrlOption(result, "readPreference", "secondary"); } else if ("secondary_preferred".equals(readPreferenceType)) { result = addUrlOption(result, "readPreference", "secondaryPreferred"); } else if ("nearest".equals(readPreferenceType)) { result = addUrlOption(result, "readPreference", "nearest"); } else { System.err.println("WARNING: Invalid readPreference: '" + readPreferenceType + "' will be ignored. " + "Must be one of [ primary | primary_preferred | " + "secondary | secondary_preferred | nearest ]"); } } return result; } /** * Adds an option to the url if it does not already contain the option. * * @param url * The URL to append the options to. * @param name * The name of the option. * @param value * The value for the option. * @return The updated URL. */ private static String addUrlOption(String url, String name, String value) { String fullName = name + "="; if (!url.contains(fullName)) { if (url.contains("?")) { return url + "&" + fullName + value; } return url + "?" + fullName + value; } return url; } /** * Hidden Constructor. */ private OptionsSupport() { // Nothing. } } ================================================ FILE: mongodb/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for MongoDB. * For additional details on using and configuring the binding see the * accompanying README.md. *

* A YCSB binding is provided for both the the * Asynchronous * Java Driver and the MongoDB Inc. * driver. *

*/ package com.yahoo.ycsb.db; ================================================ FILE: mongodb/src/main/resources/log4j.properties ================================================ # Copyright (c) 2016 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. #define the console appender log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender # now define the layout for the appender log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n # now map our console appender as a root logger, means all log messages will go # to this appender log4j.rootLogger = INFO, consoleAppender ================================================ FILE: mongodb/src/main/resources/logback.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n ================================================ FILE: mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeNoException; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.Status; import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; import java.net.InetAddress; import java.net.Socket; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * MongoDbClientTest provides runs the basic DB test cases. *

* The tests will be skipped if MongoDB is not running on port 27017 on the * local machine. See the README.md for how to get MongoDB running. *

*/ @SuppressWarnings("boxing") public abstract class AbstractDBTestCases { /** The default port for MongoDB. */ private static final int MONGODB_DEFAULT_PORT = 27017; /** * Verifies the mongod process (or some process) is running on port 27017, if * not the tests are skipped. */ @BeforeClass public static void setUpBeforeClass() { // Test if we can connect. Socket socket = null; try { // Connect socket = new Socket(InetAddress.getLocalHost(), MONGODB_DEFAULT_PORT); assertThat("Socket is not bound.", socket.getLocalPort(), not(-1)); } catch (IOException connectFailed) { assumeNoException("MongoDB is not running. Skipping tests.", connectFailed); } finally { if (socket != null) { try { socket.close(); } catch (IOException ignore) { // Ignore. } } socket = null; } } /** * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#delete} . */ @Test public void testInsertReadDelete() { final DB client = getDB(); final String table = getClass().getSimpleName(); final String id = "delete"; HashMap inserted = new HashMap(); inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); Status result = client.insert(table, id, inserted); assertThat("Insert did not return success (0).", result, is(Status.OK)); HashMap read = new HashMap(); Set keys = Collections.singleton("a"); result = client.read(table, id, keys, read); assertThat("Read did not return success (0).", result, is(Status.OK)); for (String key : keys) { ByteIterator iter = read.get(key); assertThat("Did not read the inserted field: " + key, iter, notNullValue()); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); assertFalse(iter.hasNext()); } result = client.delete(table, id); assertThat("Delete did not return success (0).", result, is(Status.OK)); read.clear(); result = client.read(table, id, null, read); assertThat("Read, after delete, did not return not found (1).", result, is(Status.NOT_FOUND)); assertThat("Found the deleted fields.", read.size(), is(0)); result = client.delete(table, id); assertThat("Delete did not return not found (1).", result, is(Status.NOT_FOUND)); } /** * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#update} . */ @Test public void testInsertReadUpdate() { DB client = getDB(); final String table = getClass().getSimpleName(); final String id = "update"; HashMap inserted = new HashMap(); inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); Status result = client.insert(table, id, inserted); assertThat("Insert did not return success (0).", result, is(Status.OK)); HashMap read = new HashMap(); Set keys = Collections.singleton("a"); result = client.read(table, id, keys, read); assertThat("Read did not return success (0).", result, is(Status.OK)); for (String key : keys) { ByteIterator iter = read.get(key); assertThat("Did not read the inserted field: " + key, iter, notNullValue()); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); assertFalse(iter.hasNext()); } HashMap updated = new HashMap(); updated.put("a", new ByteArrayByteIterator(new byte[] { 5, 6, 7, 8 })); result = client.update(table, id, updated); assertThat("Update did not return success (0).", result, is(Status.OK)); read.clear(); result = client.read(table, id, null, read); assertThat("Read, after update, did not return success (0).", result, is(Status.OK)); for (String key : keys) { ByteIterator iter = read.get(key); assertThat("Did not read the inserted field: " + key, iter, notNullValue()); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 5))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 6))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 7))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 8))); assertFalse(iter.hasNext()); } } /** * Test method for {@link DB#insert}, {@link DB#read}, and {@link DB#update} . */ @Test public void testInsertReadUpdateWithUpsert() { Properties props = new Properties(); props.setProperty("mongodb.upsert", "true"); DB client = getDB(props); final String table = getClass().getSimpleName(); final String id = "updateWithUpsert"; HashMap inserted = new HashMap(); inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); Status result = client.insert(table, id, inserted); assertThat("Insert did not return success (0).", result, is(Status.OK)); HashMap read = new HashMap(); Set keys = Collections.singleton("a"); result = client.read(table, id, keys, read); assertThat("Read did not return success (0).", result, is(Status.OK)); for (String key : keys) { ByteIterator iter = read.get(key); assertThat("Did not read the inserted field: " + key, iter, notNullValue()); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); assertFalse(iter.hasNext()); } HashMap updated = new HashMap(); updated.put("a", new ByteArrayByteIterator(new byte[] { 5, 6, 7, 8 })); result = client.update(table, id, updated); assertThat("Update did not return success (0).", result, is(Status.OK)); read.clear(); result = client.read(table, id, null, read); assertThat("Read, after update, did not return success (0).", result, is(Status.OK)); for (String key : keys) { ByteIterator iter = read.get(key); assertThat("Did not read the inserted field: " + key, iter, notNullValue()); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 5))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 6))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 7))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) 8))); assertFalse(iter.hasNext()); } } /** * Test method for {@link DB#scan}. */ @Test public void testScan() { final DB client = getDB(); final String table = getClass().getSimpleName(); // Insert a bunch of documents. for (int i = 0; i < 100; ++i) { HashMap inserted = new HashMap(); inserted.put("a", new ByteArrayByteIterator(new byte[] { (byte) (i & 0xFF), (byte) (i >> 8 & 0xFF), (byte) (i >> 16 & 0xFF), (byte) (i >> 24 & 0xFF) })); Status result = client.insert(table, padded(i), inserted); assertThat("Insert did not return success (0).", result, is(Status.OK)); } Set keys = Collections.singleton("a"); Vector> results = new Vector>(); Status result = client.scan(table, "00050", 5, null, results); assertThat("Read did not return success (0).", result, is(Status.OK)); assertThat(results.size(), is(5)); for (int i = 0; i < 5; ++i) { Map read = results.get(i); for (String key : keys) { ByteIterator iter = read.get(key); assertThat("Did not read the inserted field: " + key, iter, notNullValue()); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) ((i + 50) & 0xFF)))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) ((i + 50) >> 8 & 0xFF)))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) ((i + 50) >> 16 & 0xFF)))); assertTrue(iter.hasNext()); assertThat(iter.nextByte(), is(Byte.valueOf((byte) ((i + 50) >> 24 & 0xFF)))); assertFalse(iter.hasNext()); } } } /** * Gets the test DB. * * @return The test DB. */ protected DB getDB() { return getDB(new Properties()); } /** * Gets the test DB. * * @param props * Properties to pass to the client. * @return The test DB. */ protected abstract DB getDB(Properties props); /** * Creates a zero padded integer. * * @param i * The integer to padd. * @return The padded integer. */ private String padded(int i) { String result = String.valueOf(i); while (result.length() < 5) { result = "0" + result; } return result; } } ================================================ FILE: mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.DB; /** * AsyncMongoDbClientTest provides runs the basic workload operations. */ public class AsyncMongoDbClientTest extends MongoDbClientTest { @Override protected DB instantiateClient() { return new AsyncMongoDbClient(); } } ================================================ FILE: mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static org.junit.Assume.assumeNoException; import java.util.Properties; import org.junit.After; import com.yahoo.ycsb.DB; /** * MongoDbClientTest provides runs the basic workload operations. */ public class MongoDbClientTest extends AbstractDBTestCases { /** The client to use. */ private DB myClient = null; protected DB instantiateClient() { return new MongoDbClient(); } /** * Stops the test client. */ @After public void tearDown() { try { myClient.cleanup(); } catch (Exception error) { // Ignore. } finally { myClient = null; } } /** * {@inheritDoc} *

* Overridden to return the {@link MongoDbClient}. *

*/ @Override protected DB getDB(Properties props) { if( myClient == null ) { myClient = instantiateClient(); myClient.setProperties(props); try { myClient.init(); } catch (Exception error) { assumeNoException(error); } } return myClient; } } ================================================ FILE: mongodb/src/test/java/com/yahoo/ycsb/db/OptionsSupportTest.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static com.yahoo.ycsb.db.OptionsSupport.updateUrl; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; import java.util.Properties; import org.junit.Test; /** * OptionsSupportTest provides tests for the OptionsSupport class. * * @author rjm */ public class OptionsSupportTest { /** * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for * {@code mongodb.maxconnections}. */ @Test public void testUpdateUrlMaxConnections() { assertThat( updateUrl("mongodb://locahost:27017/", props("mongodb.maxconnections", "1234")), is("mongodb://locahost:27017/?maxPoolSize=1234")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.maxconnections", "1234")), is("mongodb://locahost:27017/?foo=bar&maxPoolSize=1234")); assertThat( updateUrl("mongodb://locahost:27017/?maxPoolSize=1", props("mongodb.maxconnections", "1234")), is("mongodb://locahost:27017/?maxPoolSize=1")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), is("mongodb://locahost:27017/?foo=bar")); } /** * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. */ @Test public void testUpdateUrlWaitQueueMultiple() { assertThat( updateUrl( "mongodb://locahost:27017/", props("mongodb.threadsAllowedToBlockForConnectionMultiplier", "1234")), is("mongodb://locahost:27017/?waitQueueMultiple=1234")); assertThat( updateUrl( "mongodb://locahost:27017/?foo=bar", props("mongodb.threadsAllowedToBlockForConnectionMultiplier", "1234")), is("mongodb://locahost:27017/?foo=bar&waitQueueMultiple=1234")); assertThat( updateUrl( "mongodb://locahost:27017/?waitQueueMultiple=1", props("mongodb.threadsAllowedToBlockForConnectionMultiplier", "1234")), is("mongodb://locahost:27017/?waitQueueMultiple=1")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), is("mongodb://locahost:27017/?foo=bar")); } /** * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. */ @Test public void testUpdateUrlWriteConcern() { assertThat( updateUrl("mongodb://locahost:27017/", props("mongodb.writeConcern", "errors_ignored")), is("mongodb://locahost:27017/?w=0")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.writeConcern", "unacknowledged")), is("mongodb://locahost:27017/?foo=bar&w=0")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.writeConcern", "acknowledged")), is("mongodb://locahost:27017/?foo=bar&w=1")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.writeConcern", "journaled")), is("mongodb://locahost:27017/?foo=bar&journal=true&j=true")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.writeConcern", "replica_acknowledged")), is("mongodb://locahost:27017/?foo=bar&w=2")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.writeConcern", "majority")), is("mongodb://locahost:27017/?foo=bar&w=majority")); // w already exists. assertThat( updateUrl("mongodb://locahost:27017/?w=1", props("mongodb.writeConcern", "acknowledged")), is("mongodb://locahost:27017/?w=1")); // Unknown options assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), is("mongodb://locahost:27017/?foo=bar")); } /** * Test method for {@link OptionsSupport#updateUrl(String, Properties)} for * {@code mongodb.threadsAllowedToBlockForConnectionMultiplier}. */ @Test public void testUpdateUrlReadPreference() { assertThat( updateUrl("mongodb://locahost:27017/", props("mongodb.readPreference", "primary")), is("mongodb://locahost:27017/?readPreference=primary")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.readPreference", "primary_preferred")), is("mongodb://locahost:27017/?foo=bar&readPreference=primaryPreferred")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.readPreference", "secondary")), is("mongodb://locahost:27017/?foo=bar&readPreference=secondary")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.readPreference", "secondary_preferred")), is("mongodb://locahost:27017/?foo=bar&readPreference=secondaryPreferred")); assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("mongodb.readPreference", "nearest")), is("mongodb://locahost:27017/?foo=bar&readPreference=nearest")); // readPreference already exists. assertThat( updateUrl("mongodb://locahost:27017/?readPreference=primary", props("mongodb.readPreference", "secondary")), is("mongodb://locahost:27017/?readPreference=primary")); // Unknown options assertThat( updateUrl("mongodb://locahost:27017/?foo=bar", props("foo", "1234")), is("mongodb://locahost:27017/?foo=bar")); } /** * Factory method for a {@link Properties} object. * * @param key * The key for the property to set. * @param value * The value for the property to set. * @return The {@link Properties} with the property added. */ private Properties props(String key, String value) { Properties props = new Properties(); props.setProperty(key, value); return props; } } ================================================ FILE: nosqldb/README.md ================================================ CONFIGURE $KVHOME is Oracle NoSQL Database package files. $KVROOT is a data directory. $YCSBHOME is a YCSB home directory. mkdir $KVROOT java -jar $KVHOME/lib/kvstore-1.2.123.jar makebootconfig \ -root $KVROOT -port 5000 -admin 5001 -host localhost \ -harange 5010,5020 java -jar $KVHOME/lib/kvstore-1.2.123.jar start -root $KVROOT java -jar $KVHOME/lib/kvstore-1.2.123.jar runadmin \ -port 5000 -host localhost -script $YCSBHOME/conf/script.txt BENCHMARK $YCSBHOME/bin/ycsb load nosqldb -P workloads/workloada $YCSBHOME/bin/ycsb run nosqldb -P workloads/workloada PROPERTIES See $YCSBHOME/conf/nosqldb.properties. STOP $ java -jar $KVHOME/lib/kvstore-1.2.123.jar stop -root $KVROOT Please refer to Oracle NoSQL Database docs here: http://docs.oracle.com/cd/NOSQL/html/index.html ================================================ FILE: nosqldb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent nosqldb-binding Oracle NoSQL Database Binding jar com.oracle.kv oracle-nosql-client 3.0.5 com.yahoo.ycsb core ${project.version} provided ================================================ FILE: nosqldb/src/main/conf/nosqldb.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Sample property file for Oracle NoSQL Database client # # Refer to the Javadoc of oracle.kv.KVStoreConfig class # for more details. # # Store name #storeName=kvstore # Comma-separated list of helper host/port pairs #helperHost=localhost:5000 # Read consistency # "ABSOLUTE" or "NONE_REQUIRED" #consistency=NONE_REQUIRED # Write durability # "COMMIT_NO_SYNC", "COMMIT_SYNC" or "COMMIT_WRITE_NO_SYNC" #durability=COMMIT_NO_SYNC # Limitations on the number of active requests to a node #requestLimit.maxActiveRequests=100 #requestLimit.requestThresholdPercent=90 #requestLimit.nodeLimitPercent=80 # Request timeout in seconds (positive integer) #requestTimeout=5 ================================================ FILE: nosqldb/src/main/conf/script.txt ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Simple configuration file; only one node in a system configure kvstore plan -execute -name "Deploy DC" deploy-datacenter "Local" plan -execute -name "Deploy n01" deploy-sn 1 localhost 5000 plan -execute -name "Deploy admin" deploy-admin 1 5001 addpool LocalPool joinpool LocalPool 1 plan -execute -name "Deploy the store" deploy-store LocalPool 1 100 quit ================================================ FILE: nosqldb/src/main/java/com/yahoo/ycsb/db/NoSqlDbClient.java ================================================ /** * Copyright (c) 2012 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.SortedMap; import java.util.Vector; import java.util.concurrent.TimeUnit; import oracle.kv.Consistency; import oracle.kv.Durability; import oracle.kv.FaultException; import oracle.kv.KVStore; import oracle.kv.KVStoreConfig; import oracle.kv.KVStoreFactory; import oracle.kv.Key; import oracle.kv.RequestLimitConfig; import oracle.kv.Value; import oracle.kv.ValueVersion; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; /** * A database interface layer for Oracle NoSQL Database. */ public class NoSqlDbClient extends DB { private KVStore store; private int getPropertyInt(Properties properties, String key, int defaultValue) throws DBException { String p = properties.getProperty(key); int i = defaultValue; if (p != null) { try { i = Integer.parseInt(p); } catch (NumberFormatException e) { throw new DBException("Illegal number format in " + key + " property"); } } return i; } @Override public void init() throws DBException { Properties properties = getProperties(); /* Mandatory properties */ String storeName = properties.getProperty("storeName", "kvstore"); String[] helperHosts = properties.getProperty("helperHost", "localhost:5000").split(","); KVStoreConfig config = new KVStoreConfig(storeName, helperHosts); /* Optional properties */ String p; p = properties.getProperty("consistency"); if (p != null) { if (p.equalsIgnoreCase("ABSOLUTE")) { config.setConsistency(Consistency.ABSOLUTE); } else if (p.equalsIgnoreCase("NONE_REQUIRED")) { config.setConsistency(Consistency.NONE_REQUIRED); } else { throw new DBException("Illegal value in consistency property"); } } p = properties.getProperty("durability"); if (p != null) { if (p.equalsIgnoreCase("COMMIT_NO_SYNC")) { config.setDurability(Durability.COMMIT_NO_SYNC); } else if (p.equalsIgnoreCase("COMMIT_SYNC")) { config.setDurability(Durability.COMMIT_SYNC); } else if (p.equalsIgnoreCase("COMMIT_WRITE_NO_SYNC")) { config.setDurability(Durability.COMMIT_WRITE_NO_SYNC); } else { throw new DBException("Illegal value in durability property"); } } int maxActiveRequests = getPropertyInt(properties, "requestLimit.maxActiveRequests", RequestLimitConfig.DEFAULT_MAX_ACTIVE_REQUESTS); int requestThresholdPercent = getPropertyInt(properties, "requestLimit.requestThresholdPercent", RequestLimitConfig.DEFAULT_REQUEST_THRESHOLD_PERCENT); int nodeLimitPercent = getPropertyInt(properties, "requestLimit.nodeLimitPercent", RequestLimitConfig.DEFAULT_NODE_LIMIT_PERCENT); RequestLimitConfig requestLimitConfig; /* * It is said that the constructor could throw NodeRequestLimitException in * Javadoc, the exception is not provided */ // try { requestLimitConfig = new RequestLimitConfig(maxActiveRequests, requestThresholdPercent, nodeLimitPercent); // } catch (NodeRequestLimitException e) { // throw new DBException(e); // } config.setRequestLimit(requestLimitConfig); p = properties.getProperty("requestTimeout"); if (p != null) { long timeout = 1; try { timeout = Long.parseLong(p); } catch (NumberFormatException e) { throw new DBException( "Illegal number format in requestTimeout property"); } try { // TODO Support other TimeUnit config.setRequestTimeout(timeout, TimeUnit.SECONDS); } catch (IllegalArgumentException e) { throw new DBException(e); } } try { store = KVStoreFactory.getStore(config); } catch (FaultException e) { throw new DBException(e); } } @Override public void cleanup() throws DBException { store.close(); } /** * Create a key object. We map "table" and (YCSB's) "key" to a major component * of the oracle.kv.Key, and "field" to a minor component. * * @return An oracle.kv.Key object. */ private static Key createKey(String table, String key, String field) { List majorPath = new ArrayList(); majorPath.add(table); majorPath.add(key); if (field == null) { return Key.createKey(majorPath); } return Key.createKey(majorPath, field); } private static Key createKey(String table, String key) { return createKey(table, key, null); } private static String getFieldFromKey(Key key) { return key.getMinorPath().get(0); } @Override public Status read(String table, String key, Set fields, Map result) { Key kvKey = createKey(table, key); SortedMap kvResult; try { kvResult = store.multiGet(kvKey, null, null); } catch (FaultException e) { System.err.println(e); return Status.ERROR; } for (Map.Entry entry : kvResult.entrySet()) { /* If fields is null, read all fields */ String field = getFieldFromKey(entry.getKey()); if (fields != null && !fields.contains(field)) { continue; } result.put(field, new ByteArrayByteIterator(entry.getValue().getValue().getValue())); } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { System.err.println("Oracle NoSQL Database does not support Scan semantics"); return Status.ERROR; } @Override public Status update(String table, String key, Map values) { for (Map.Entry entry : values.entrySet()) { Key kvKey = createKey(table, key, entry.getKey()); Value kvValue = Value.createValue(entry.getValue().toArray()); try { store.put(kvKey, kvValue); } catch (FaultException e) { System.err.println(e); return Status.ERROR; } } return Status.OK; } @Override public Status insert(String table, String key, Map values) { return update(table, key, values); } @Override public Status delete(String table, String key) { Key kvKey = createKey(table, key); try { store.multiDelete(kvKey, null, null); } catch (FaultException e) { System.err.println(e); return Status.ERROR; } return Status.OK; } } ================================================ FILE: nosqldb/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. See accompanying LICENSE file. */ /** * The YCSB binding for Oracle * 's NoSQL DB. */ package com.yahoo.ycsb.db; ================================================ FILE: orientdb/README.md ================================================ ## Quick Start This section describes how to run YCSB on OrientDB running locally. ### 1. Set Up YCSB Clone the YCSB git repository and compile: git clone https://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package ### 2. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load orientdb -s -P workloads/workloada Then, run the workload: ./bin/ycsb run orientdb -s -P workloads/workloada See the next section for the list of configuration parameters for OrientDB. ## DB creation with the OrientDBClient This client will create a database for you if the connection database you specify does not exists. You can also specify connection information to a preexisting database. You can use the ```orientdb.newdb=true``` property to allow this client to drop and create a new database instance during the ```load``` phase. NOTE: understand that using the ```orientdb.newdb=true``` property will drop and recreate databases even if it was a preexisting instance. WARNING: Creating a new database will be done safely with multiple threads on a single YCSB instance, but is not guaranteed to work when launching multiple YCSB instances. In that scenario it is suggested that you create the db before hand, or run the ```load``` phase with a single YCSB instance. ## OrientDB Configuration Parameters * ```orientdb.url``` - (required) The address to your database. * Supported storage types: memory, plocal, remote * EX. ```plocal:/path/to/database``` * ```orientdb.user``` - The user to connect to the database with. * Default: ```admin``` * ```orientdb.password``` - The password to connect to the database with. * Default: ```admin``` * ```orientdb.newdb``` - Overwrite the database if it already exists. * Only effects the ```load``` phase. * Default: ```false``` * ```orientdb.remote.storagetype``` - Storage type of the database on remote server * This is only required if using a ```remote:``` connection url ## Known Issues * There is a performance issue around the scan operation. This binding uses OIndex.iterateEntriesMajor() which will return unnecessarily large iterators. This has a performance impact as the recordcount goes up. There are ideas in the works to fix it, track it here: [#568](https://github.com/brianfrankcooper/YCSB/issues/568). * Iterator methods needed to perform scans are Unsupported in the OrientDB API for remote database connections and so will return NOT_IMPLEMENTED status if attempted. ================================================ FILE: orientdb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent orientdb-binding OrientDB Binding jar sonatype-nexus-snapshots Sonatype Nexus Snapshots https://oss.sonatype.org/content/repositories/snapshots true com.yahoo.ycsb core ${project.version} provided com.orientechnologies orientdb-client ${orientdb.version} junit junit 4.12 test org.slf4j slf4j-log4j12 1.7.10 ================================================ FILE: orientdb/src/main/java/com/yahoo/ycsb/db/OrientDBClient.java ================================================ /** * Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.orientechnologies.orient.client.remote.OServerAdmin; import com.orientechnologies.orient.core.config.OGlobalConfiguration; import com.orientechnologies.orient.core.db.OPartitionedDatabasePool; import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; import com.orientechnologies.orient.core.dictionary.ODictionary; import com.orientechnologies.orient.core.exception.OConcurrentModificationException; import com.orientechnologies.orient.core.index.OIndexCursor; import com.orientechnologies.orient.core.record.ORecord; import com.orientechnologies.orient.core.record.impl.ODocument; import com.yahoo.ycsb.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; /** * OrientDB client for YCSB framework. */ public class OrientDBClient extends DB { private static final String URL_PROPERTY = "orientdb.url"; private static final String URL_PROPERTY_DEFAULT = "plocal:." + File.separator + "target" + File.separator + "databases" + File.separator + "ycsb"; private static final String USER_PROPERTY = "orientdb.user"; private static final String USER_PROPERTY_DEFAULT = "admin"; private static final String PASSWORD_PROPERTY = "orientdb.password"; private static final String PASSWORD_PROPERTY_DEFAULT = "admin"; private static final String NEWDB_PROPERTY = "orientdb.newdb"; private static final String NEWDB_PROPERTY_DEFAULT = "false"; private static final String STORAGE_TYPE_PROPERTY = "orientdb.remote.storagetype"; private static final String ORIENTDB_DOCUMENT_TYPE = "document"; private static final String CLASS = "usertable"; private static final Lock INIT_LOCK = new ReentrantLock(); private static boolean dbChecked = false; private static volatile OPartitionedDatabasePool databasePool; private static boolean initialized = false; private static int clientCounter = 0; private boolean isRemote = false; private static final Logger LOG = LoggerFactory.getLogger(OrientDBClient.class); /** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per client thread. */ public void init() throws DBException { // initialize OrientDB driver final Properties props = getProperties(); String url = props.getProperty(URL_PROPERTY, URL_PROPERTY_DEFAULT); String user = props.getProperty(USER_PROPERTY, USER_PROPERTY_DEFAULT); String password = props.getProperty(PASSWORD_PROPERTY, PASSWORD_PROPERTY_DEFAULT); Boolean newdb = Boolean.parseBoolean(props.getProperty(NEWDB_PROPERTY, NEWDB_PROPERTY_DEFAULT)); String remoteStorageType = props.getProperty(STORAGE_TYPE_PROPERTY); INIT_LOCK.lock(); try { clientCounter++; if (!initialized) { OGlobalConfiguration.dumpConfiguration(System.out); LOG.info("OrientDB loading database url = " + url); ODatabaseDocumentTx db = new ODatabaseDocumentTx(url); if (db.getStorage().isRemote()) { isRemote = true; } if (!dbChecked) { if (!isRemote) { if (newdb) { if (db.exists()) { db.open(user, password); LOG.info("OrientDB drop and recreate fresh db"); db.drop(); } db.create(); } else { if (!db.exists()) { LOG.info("OrientDB database not found, creating fresh db"); db.create(); } } } else { OServerAdmin server = new OServerAdmin(url).connect(user, password); if (remoteStorageType == null) { throw new DBException( "When connecting to a remote OrientDB instance, " + "specify a database storage type (plocal or memory) with " + STORAGE_TYPE_PROPERTY); } if (newdb) { if (server.existsDatabase()) { LOG.info("OrientDB drop and recreate fresh db"); server.dropDatabase(remoteStorageType); } server.createDatabase(db.getName(), ORIENTDB_DOCUMENT_TYPE, remoteStorageType); } else { if (!server.existsDatabase()) { LOG.info("OrientDB database not found, creating fresh db"); server.createDatabase(server.getURL(), ORIENTDB_DOCUMENT_TYPE, remoteStorageType); } } server.close(); } dbChecked = true; } if (db.isClosed()) { db.open(user, password); } if (!db.getMetadata().getSchema().existsClass(CLASS)) { db.getMetadata().getSchema().createClass(CLASS); } db.close(); if (databasePool == null) { databasePool = new OPartitionedDatabasePool(url, user, password); } initialized = true; } } catch (Exception e) { LOG.error("Could not initialize OrientDB connection pool for Loader: " + e.toString()); e.printStackTrace(); } finally { INIT_LOCK.unlock(); } } OPartitionedDatabasePool getDatabasePool() { return databasePool; } @Override public void cleanup() throws DBException { INIT_LOCK.lock(); try { clientCounter--; if (clientCounter == 0) { databasePool.close(); } databasePool = null; initialized = false; } finally { INIT_LOCK.unlock(); } } @Override public Status insert(String table, String key, Map values) { try (ODatabaseDocumentTx db = databasePool.acquire()) { final ODocument document = new ODocument(CLASS); for (Map.Entry entry : StringByteIterator.getStringMap(values).entrySet()) { document.field(entry.getKey(), entry.getValue()); } document.save(); final ODictionary dictionary = db.getMetadata().getIndexManager().getDictionary(); dictionary.put(key, document); return Status.OK; } catch (Exception e) { e.printStackTrace(); } return Status.ERROR; } @Override public Status delete(String table, String key) { while (true) { try (ODatabaseDocumentTx db = databasePool.acquire()) { final ODictionary dictionary = db.getMetadata().getIndexManager().getDictionary(); dictionary.remove(key); return Status.OK; } catch (OConcurrentModificationException cme) { continue; } catch (Exception e) { e.printStackTrace(); return Status.ERROR; } } } @Override public Status read(String table, String key, Set fields, Map result) { try (ODatabaseDocumentTx db = databasePool.acquire()) { final ODictionary dictionary = db.getMetadata().getIndexManager().getDictionary(); final ODocument document = dictionary.get(key); if (document != null) { if (fields != null) { for (String field : fields) { result.put(field, new StringByteIterator((String) document.field(field))); } } else { for (String field : document.fieldNames()) { result.put(field, new StringByteIterator((String) document.field(field))); } } return Status.OK; } } catch (Exception e) { e.printStackTrace(); } return Status.ERROR; } @Override public Status update(String table, String key, Map values) { while (true) { try (ODatabaseDocumentTx db = databasePool.acquire()) { final ODictionary dictionary = db.getMetadata().getIndexManager().getDictionary(); final ODocument document = dictionary.get(key); if (document != null) { for (Map.Entry entry : StringByteIterator.getStringMap(values).entrySet()) { document.field(entry.getKey(), entry.getValue()); } document.save(); return Status.OK; } } catch (OConcurrentModificationException cme) { continue; } catch (Exception e) { e.printStackTrace(); return Status.ERROR; } } } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { if (isRemote) { // Iterator methods needed for scanning are Unsupported for remote database connections. LOG.warn("OrientDB scan operation is not implemented for remote database connections."); return Status.NOT_IMPLEMENTED; } try (ODatabaseDocumentTx db = databasePool.acquire()) { final ODictionary dictionary = db.getMetadata().getIndexManager().getDictionary(); final OIndexCursor entries = dictionary.getIndex().iterateEntriesMajor(startkey, true, true); int currentCount = 0; while (entries.hasNext()) { final ODocument document = entries.next().getRecord(); final HashMap map = new HashMap<>(); result.add(map); if (fields != null) { for (String field : fields) { map.put(field, new StringByteIterator((String) document.field(field))); } } else { for (String field : document.fieldNames()) { map.put(field, new StringByteIterator((String) document.field(field))); } } currentCount++; if (currentCount >= recordcount) { break; } } return Status.OK; } catch (Exception e) { e.printStackTrace(); } return Status.ERROR; } } ================================================ FILE: orientdb/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2015 - 2016, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for OrientDB. */ package com.yahoo.ycsb.db; ================================================ FILE: orientdb/src/main/resources/log4j.properties ================================================ # Root logger option log4j.rootLogger=INFO, stderr # Direct log messages to stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.Target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n ================================================ FILE: orientdb/src/test/java/com/yahoo/ycsb/db/OrientDBClientTest.java ================================================ /** * Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.orientechnologies.orient.core.db.OPartitionedDatabasePool; import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx; import com.orientechnologies.orient.core.dictionary.ODictionary; import com.orientechnologies.orient.core.record.ORecord; import com.orientechnologies.orient.core.record.impl.ODocument; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.StringByteIterator; import org.junit.*; import java.util.*; import static org.junit.Assert.*; /** * Created by kruthar on 12/29/15. */ public class OrientDBClientTest { // TODO: This must be copied because it is private in OrientDBClient, but this should defer to table property. private static final String CLASS = "usertable"; private static final int FIELD_LENGTH = 32; private static final String FIELD_PREFIX = "FIELD"; private static final String KEY_PREFIX = "user"; private static final int NUM_FIELDS = 3; private static final String TEST_DB_URL = "memory:test"; private static OrientDBClient orientDBClient = null; @Before public void setup() throws DBException { orientDBClient = new OrientDBClient(); Properties p = new Properties(); // TODO: Extract the property names into final variables in OrientDBClient p.setProperty("orientdb.url", TEST_DB_URL); orientDBClient.setProperties(p); orientDBClient.init(); } @After public void teardown() throws DBException { if (orientDBClient != null) { orientDBClient.cleanup(); } } /* This is a copy of buildDeterministicValue() from core:com.yahoo.ycsb.workloads.CoreWorkload.java. That method is neither public nor static so we need a copy. */ private String buildDeterministicValue(String key, String fieldkey) { int size = FIELD_LENGTH; StringBuilder sb = new StringBuilder(size); sb.append(key); sb.append(':'); sb.append(fieldkey); while (sb.length() < size) { sb.append(':'); sb.append(sb.toString().hashCode()); } sb.setLength(size); return sb.toString(); } /* Inserts a row of deterministic values for the given insertKey using the orientDBClient. */ private Map insertRow(String insertKey) { HashMap insertMap = new HashMap<>(); for (int i = 0; i < 3; i++) { insertMap.put(FIELD_PREFIX + i, new StringByteIterator(buildDeterministicValue(insertKey, FIELD_PREFIX + i))); } orientDBClient.insert(CLASS, insertKey, insertMap); return insertMap; } @Test public void insertTest() { String insertKey = "user0"; Map insertMap = insertRow(insertKey); OPartitionedDatabasePool pool = orientDBClient.getDatabasePool(); try(ODatabaseDocumentTx db = pool.acquire()) { ODictionary dictionary = db.getDictionary(); ODocument result = dictionary.get(insertKey); assertTrue("Assert a row was inserted.", result != null); for (int i = 0; i < NUM_FIELDS; i++) { assertEquals("Assert all inserted columns have correct values.", result.field(FIELD_PREFIX + i), insertMap.get(FIELD_PREFIX + i).toString()); } } } @Test public void updateTest() { String preupdateString = "preupdate"; String user0 = "user0"; String user1 = "user1"; String user2 = "user2"; OPartitionedDatabasePool pool = orientDBClient.getDatabasePool(); try(ODatabaseDocumentTx db = pool.acquire()) { // Manually insert three documents for (String key : Arrays.asList(user0, user1, user2)) { ODocument doc = new ODocument(CLASS); for (int i = 0; i < NUM_FIELDS; i++) { doc.field(FIELD_PREFIX + i, preupdateString); } doc.save(); ODictionary dictionary = db.getDictionary(); dictionary.put(key, doc); } } HashMap updateMap = new HashMap<>(); for (int i = 0; i < NUM_FIELDS; i++) { updateMap.put(FIELD_PREFIX + i, new StringByteIterator(buildDeterministicValue(user1, FIELD_PREFIX + i))); } orientDBClient.update(CLASS, user1, updateMap); try(ODatabaseDocumentTx db = pool.acquire()) { ODictionary dictionary = db.getDictionary(); // Ensure that user0 record was not changed ODocument result = dictionary.get(user0); for (int i = 0; i < NUM_FIELDS; i++) { assertEquals("Assert first row fields contain preupdateString", result.field(FIELD_PREFIX + i), preupdateString); } // Check that all the columns have expected values for user1 record result = dictionary.get(user1); for (int i = 0; i < NUM_FIELDS; i++) { assertEquals("Assert updated row fields are correct", result.field(FIELD_PREFIX + i), updateMap.get(FIELD_PREFIX + i).toString()); } // Ensure that user2 record was not changed result = dictionary.get(user2); for (int i = 0; i < NUM_FIELDS; i++) { assertEquals("Assert third row fields contain preupdateString", result.field(FIELD_PREFIX + i), preupdateString); } } } @Test public void readTest() { String insertKey = "user0"; Map insertMap = insertRow(insertKey); HashSet readFields = new HashSet<>(); HashMap readResultMap = new HashMap<>(); // Test reading a single field readFields.add("FIELD0"); orientDBClient.read(CLASS, insertKey, readFields, readResultMap); assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size()); for (String field : readFields) { assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString()); } readResultMap = new HashMap<>(); // Test reading all fields readFields.add("FIELD1"); readFields.add("FIELD2"); orientDBClient.read(CLASS, insertKey, readFields, readResultMap); assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size()); for (String field : readFields) { assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString()); } } @Test public void deleteTest() { String user0 = "user0"; String user1 = "user1"; String user2 = "user2"; insertRow(user0); insertRow(user1); insertRow(user2); orientDBClient.delete(CLASS, user1); OPartitionedDatabasePool pool = orientDBClient.getDatabasePool(); try(ODatabaseDocumentTx db = pool.acquire()) { ODictionary dictionary = db.getDictionary(); assertNotNull("Assert user0 still exists", dictionary.get(user0)); assertNull("Assert user1 does not exist", dictionary.get(user1)); assertNotNull("Assert user2 still exists", dictionary.get(user2)); } } @Test public void scanTest() { Map> keyMap = new HashMap<>(); for (int i = 0; i < 5; i++) { String insertKey = KEY_PREFIX + i; keyMap.put(insertKey, insertRow(insertKey)); } Set fieldSet = new HashSet<>(); fieldSet.add("FIELD0"); fieldSet.add("FIELD1"); int startIndex = 0; int resultRows = 3; Vector> resultVector = new Vector<>(); orientDBClient.scan(CLASS, KEY_PREFIX + startIndex, resultRows, fieldSet, resultVector); // Check the resultVector is the correct size assertEquals("Assert the correct number of results rows were returned", resultRows, resultVector.size()); int testIndex = startIndex; // Check each vector row to make sure we have the correct fields for (HashMap result : resultVector) { assertEquals("Assert that this row has the correct number of fields", fieldSet.size(), result.size()); for (String field : fieldSet) { assertEquals("Assert this field is correct in this row", keyMap.get(KEY_PREFIX + testIndex).get(field).toString(), result.get(field).toString()); } testIndex++; } } } ================================================ FILE: pom.xml ================================================ 4.0.0 com.yahoo.ycsb root 0.14.0-SNAPSHOT pom YCSB Root This is the top level project that builds, packages the core and all the DB bindings for YCSB infrastructure. scm:git:git://github.com/brianfrankcooper/YCSB.git master https://github.com/brianfrankcooper/YCSB com.puppycrawl.tools checkstyle 7.7.1 org.jdom jdom 1.1 com.google.collections google-collections 1.0 org.slf4j slf4j-api 1.6.4 2.5.5 2.10 1.7.1 0.98.14-hadoop2 1.0.2 1.2.5 1.6.6 1.7.3 1.8.1 3.0.0 1.2.0 1.8.1 0.9.7 7.2.2.Final 1.1.0 2.1.1 3.0.3 2.0.1 2.2.10 2.0.0 1.10.20 0.81 UTF-8 0.8.0 0.9.5.6 5.5.1 1.4.10 2.3.1 1.6.5 2.0.5 3.1.2 5.5.3 6.4.1 2.7.3 4.1.7 4.0.0 0.24.0-beta core binding-parent distribution accumulo1.6 accumulo1.7 accumulo1.8 aerospike arangodb arangodb3 asynchbase azuredocumentdb azuretablestorage cassandra cloudspanner couchbase couchbase2 dynamodb elasticsearch elasticsearch5 geode googlebigtable googledatastore hbase098 hbase10 hbase12 hypertable infinispan jdbc kudu memcached mongodb nosqldb orientdb rados redis rest riak s3 solr solr6 tarantool org.apache.maven.plugins maven-checkstyle-plugin 2.16 org.apache.maven.plugins maven-enforcer-plugin 3.0.0-M1 enforce-maven enforce 3.1.0 org.apache.maven.plugins maven-compiler-plugin 3.7.0 1.7 1.7 org.apache.maven.plugins maven-checkstyle-plugin validate validate check checkstyle.xml ================================================ FILE: rados/README.md ================================================ ## Quick Start This section describes how to run YCSB on RADOS of Ceph. ### 1. Start RADOS After you start your Ceph cluster, check your cluster’s health first. You can check on the health of your cluster with the following: ceph health ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn clean package You can compile only RADOS-binding, EG: mvn -pl com.yahoo.ycsb:rados-binding -am clean package You can skip the test, EG: mvn -pl com.yahoo.ycsb:rados-binding -am clean package -DskipTests ### 4. Configuration Parameters - `rados.configfile` - The path of the Ceph configuration file - Default value is '/etc/ceph/ceph.conf' - `rados.id` - The user id to access the RADOS service - Default value is 'admin' - `rados.pool` - The pool name to be used for benchmark - Default value is 'data' You can set configurations with the shell command, EG: ./bin/ycsb load rados -s -P workloads/workloada -p "rados.configfile=/etc/ceph/ceph.conf" -p "rados.id=admin" -p "rados.pool=data" > outputLoad.txt ### 5. Load data and run tests Load the data: ./bin/ycsb load rados -s -P workloads/workloada > outputLoad.txt Run the workload test: ./bin/ycsb run rados -s -P workloads/workloada > outputRun.txt ================================================ FILE: rados/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent rados-binding rados of Ceph FS binding jar com.ceph rados ${rados.version} com.yahoo.ycsb core ${project.version} provided org.json json ${json.version} net.java.dev.jna jna 4.2.2 junit junit 4.12 test 0.2.0 20160212 ================================================ FILE: rados/src/main/java/com/yahoo/ycsb/db/RadosClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.ceph.rados.Rados; import com.ceph.rados.IoCTX; import com.ceph.rados.jna.RadosObjectInfo; import com.ceph.rados.ReadOp; import com.ceph.rados.ReadOp.ReadResult; import com.ceph.rados.exceptions.RadosException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import java.io.File; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Vector; import org.json.JSONObject; /** * YCSB binding for RADOS of Ceph. * * See {@code rados/README.md} for details. */ public class RadosClient extends DB { private Rados rados; private IoCTX ioctx; public static final String CONFIG_FILE_PROPERTY = "rados.configfile"; public static final String CONFIG_FILE_DEFAULT = "/etc/ceph/ceph.conf"; public static final String ID_PROPERTY = "rados.id"; public static final String ID_DEFAULT = "admin"; public static final String POOL_PROPERTY = "rados.pool"; public static final String POOL_DEFAULT = "data"; private boolean isInited = false; public void init() throws DBException { Properties props = getProperties(); String configfile = props.getProperty(CONFIG_FILE_PROPERTY); if (configfile == null) { configfile = CONFIG_FILE_DEFAULT; } String id = props.getProperty(ID_PROPERTY); if (id == null) { id = ID_DEFAULT; } String pool = props.getProperty(POOL_PROPERTY); if (pool == null) { pool = POOL_DEFAULT; } // try { // } catch (UnsatisfiedLinkError e) { // throw new DBException("RADOS library is not loaded."); // } rados = new Rados(id); try { rados.confReadFile(new File(configfile)); rados.connect(); ioctx = rados.ioCtxCreate(pool); } catch (RadosException e) { throw new DBException(e.getMessage() + ": " + e.getReturnValue()); } isInited = true; } public void cleanup() throws DBException { if (isInited) { rados.shutDown(); rados.ioCtxDestroy(ioctx); isInited = false; } } @Override public Status read(String table, String key, Set fields, Map result) { byte[] buffer; try { RadosObjectInfo info = ioctx.stat(key); buffer = new byte[(int)info.getSize()]; ReadOp rop = ioctx.readOpCreate(); ReadResult readResult = rop.queueRead(0, info.getSize()); // TODO: more size than byte length possible; // rop.operate(key, Rados.OPERATION_NOFLAG); // for rados-java 0.3.0 rop.operate(key, 0); // readResult.raiseExceptionOnError("Error ReadOP(%d)", readResult.getRVal()); // for rados-java 0.3.0 if (readResult.getRVal() < 0) { throw new RadosException("Error ReadOP", readResult.getRVal()); } if (info.getSize() != readResult.getBytesRead()) { return new Status("ERROR", "Error the object size read"); } readResult.getBuffer().get(buffer); } catch (RadosException e) { return new Status("ERROR-" + e.getReturnValue(), e.getMessage()); } JSONObject json = new JSONObject(new String(buffer, java.nio.charset.StandardCharsets.UTF_8)); Set fieldsToReturn = (fields == null ? json.keySet() : fields); for (String name : fieldsToReturn) { result.put(name, new StringByteIterator(json.getString(name))); } return result.isEmpty() ? Status.ERROR : Status.OK; } @Override public Status insert(String table, String key, Map values) { JSONObject json = new JSONObject(); for (final Entry e : values.entrySet()) { json.put(e.getKey(), e.getValue().toString()); } try { ioctx.write(key, json.toString()); } catch (RadosException e) { return new Status("ERROR-" + e.getReturnValue(), e.getMessage()); } return Status.OK; } @Override public Status delete(String table, String key) { try { ioctx.remove(key); } catch (RadosException e) { return new Status("ERROR-" + e.getReturnValue(), e.getMessage()); } return Status.OK; } @Override public Status update(String table, String key, Map values) { Status rtn = delete(table, key); if (rtn.equals(Status.OK)) { return insert(table, key, values); } return rtn; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { return Status.NOT_IMPLEMENTED; } } ================================================ FILE: rados/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for RADOS of Ceph. */ package com.yahoo.ycsb.db; ================================================ FILE: rados/src/test/java/com/yahoo/ycsb/db/RadosClientTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import static org.junit.Assert.assertEquals; import static org.junit.Assume.assumeNoException; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.junit.AfterClass; import org.junit.After; import org.junit.BeforeClass; import org.junit.Before; import org.junit.Test; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.UUID; /** * Test for the binding of RADOS of Ceph. * * See {@code rados/README.md} for details. */ public class RadosClientTest { private static RadosClient radosclient; public static final String POOL_PROPERTY = "rados.pool"; public static final String POOL_TEST = "rbd"; private static final String TABLE_NAME = "table0"; private static final String KEY0 = "key0"; private static final String KEY1 = "key1"; private static final String KEY2 = "key2"; private static final HashMap DATA; private static final HashMap DATA_UPDATED; static { DATA = new HashMap(10); DATA_UPDATED = new HashMap(10); for (int i = 0; i < 10; i++) { String key = "key" + UUID.randomUUID(); DATA.put(key, new StringByteIterator("data" + UUID.randomUUID())); DATA_UPDATED.put(key, new StringByteIterator("data" + UUID.randomUUID())); } } @BeforeClass public static void setupClass() throws DBException { radosclient = new RadosClient(); Properties p = new Properties(); p.setProperty(POOL_PROPERTY, POOL_TEST); try { radosclient.setProperties(p); radosclient.init(); } catch (DBException|UnsatisfiedLinkError e) { assumeNoException("Ceph cluster is not running. Skipping tests.", e); } } @AfterClass public static void teardownClass() throws DBException { if (radosclient != null) { radosclient.cleanup(); } } @Before public void setUp() { radosclient.insert(TABLE_NAME, KEY0, DATA); } @After public void tearDown() { radosclient.delete(TABLE_NAME, KEY0); } @Test public void insertTest() { Status result = radosclient.insert(TABLE_NAME, KEY1, DATA); assertEquals(Status.OK, result); } @Test public void updateTest() { radosclient.insert(TABLE_NAME, KEY2, DATA); Status result = radosclient.update(TABLE_NAME, KEY2, DATA_UPDATED); assertEquals(Status.OK, result); HashMap ret = new HashMap(10); radosclient.read(TABLE_NAME, KEY2, DATA.keySet(), ret); compareMap(DATA_UPDATED, ret); radosclient.delete(TABLE_NAME, KEY2); } @Test public void readTest() { HashMap ret = new HashMap(10); Status result = radosclient.read(TABLE_NAME, KEY0, DATA.keySet(), ret); assertEquals(Status.OK, result); compareMap(DATA, ret); } private void compareMap(HashMap src, HashMap dest) { assertEquals(src.size(), dest.size()); Set setSrc = src.entrySet(); Iterator itSrc = setSrc.iterator(); for (int i = 0; i < 10; i++) { Map.Entry entrySrc = itSrc.next(); assertEquals(entrySrc.getValue().toString(), dest.get(entrySrc.getKey()).toString()); } } @Test public void deleteTest() { Status result = radosclient.delete(TABLE_NAME, KEY0); assertEquals(Status.OK, result); } } ================================================ FILE: redis/README.md ================================================ ## Quick Start This section describes how to run YCSB on Redis. ### 1. Start Redis ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:redis-binding -am clean package ### 4. Provide Redis Connection Parameters Set the host, port, and password (do not redis auth is not turned on) in the workload you plan to run. - `redis.host` - `redis.port` - `redis.password` Or, you can set configs with the shell command, EG: ./bin/ycsb load redis -s -P workloads/workloada -p "redis.host=127.0.0.1" -p "redis.port=6379" > outputLoad.txt ### 5. Load data and run tests Load the data: ./bin/ycsb load redis -s -P workloads/workloada > outputLoad.txt Run the workload test: ./bin/ycsb run redis -s -P workloads/workloada > outputRun.txt ================================================ FILE: redis/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent redis-binding Redis DB Binding jar redis.clients jedis ${redis.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: redis/src/main/java/com/yahoo/ycsb/db/RedisClient.java ================================================ /** * Copyright (c) 2012 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * Redis client binding for YCSB. * * All YCSB records are mapped to a Redis *hash field*. For scanning * operations, all keys are saved (by an arbitrary hash) in a sorted set. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import redis.clients.jedis.Jedis; import redis.clients.jedis.Protocol; import java.util.HashMap; import java.util.Map; import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.Vector; /** * YCSB binding for Redis. * * See {@code redis/README.md} for details. */ public class RedisClient extends DB { private Jedis jedis; public static final String HOST_PROPERTY = "redis.host"; public static final String PORT_PROPERTY = "redis.port"; public static final String PASSWORD_PROPERTY = "redis.password"; public static final String INDEX_KEY = "_indices"; public void init() throws DBException { Properties props = getProperties(); int port; String portString = props.getProperty(PORT_PROPERTY); if (portString != null) { port = Integer.parseInt(portString); } else { port = Protocol.DEFAULT_PORT; } String host = props.getProperty(HOST_PROPERTY); jedis = new Jedis(host, port); jedis.connect(); String password = props.getProperty(PASSWORD_PROPERTY); if (password != null) { jedis.auth(password); } } public void cleanup() throws DBException { jedis.disconnect(); } /* * Calculate a hash for a key to store it in an index. The actual return value * of this function is not interesting -- it primarily needs to be fast and * scattered along the whole space of doubles. In a real world scenario one * would probably use the ASCII values of the keys. */ private double hash(String key) { return key.hashCode(); } // XXX jedis.select(int index) to switch to `table` @Override public Status read(String table, String key, Set fields, Map result) { if (fields == null) { StringByteIterator.putAllAsByteIterators(result, jedis.hgetAll(key)); } else { String[] fieldArray = (String[]) fields.toArray(new String[fields.size()]); List values = jedis.hmget(key, fieldArray); Iterator fieldIterator = fields.iterator(); Iterator valueIterator = values.iterator(); while (fieldIterator.hasNext() && valueIterator.hasNext()) { result.put(fieldIterator.next(), new StringByteIterator(valueIterator.next())); } assert !fieldIterator.hasNext() && !valueIterator.hasNext(); } return result.isEmpty() ? Status.ERROR : Status.OK; } @Override public Status insert(String table, String key, Map values) { if (jedis.hmset(key, StringByteIterator.getStringMap(values)) .equals("OK")) { jedis.zadd(INDEX_KEY, hash(key), key); return Status.OK; } return Status.ERROR; } @Override public Status delete(String table, String key) { return jedis.del(key) == 0 && jedis.zrem(INDEX_KEY, key) == 0 ? Status.ERROR : Status.OK; } @Override public Status update(String table, String key, Map values) { return jedis.hmset(key, StringByteIterator.getStringMap(values)) .equals("OK") ? Status.OK : Status.ERROR; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { Set keys = jedis.zrangeByScore(INDEX_KEY, hash(startkey), Double.POSITIVE_INFINITY, 0, recordcount); HashMap values; for (String key : keys) { values = new HashMap(); read(table, key, fields, values); result.add(values); } return Status.OK; } } ================================================ FILE: redis/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Redis. */ package com.yahoo.ycsb.db; ================================================ FILE: rest/README.md ================================================ ## Quick Start This section describes how to run YCSB to benchmark HTTP RESTful webservices. The aim of the rest binding is to benchmark the performance of any sepecific HTTP RESTful webservices with real life (production) dataset. This must not be confused with benchmarking various webservers (like Apache Tomcat, Nginx, Jetty) using a dummy dataset. ### 1. Set Up YCSB Clone the YCSB git repository and compile: git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:rest-binding -am clean package ### 2. Set Up an HTTP Web Service There must be a running HTTP RESTful webservice accesible from the instance on which YCSB is running. If the webservice is running on the local instance default HTTP port 80, it's base URL will look like http://127.0.0.1:80/{service_endpoint}. The rest binding assumes that the webservice to be benchmarked already has a valid dataset. THe rest module has been designed in this way for two reasons: 1. The performance of most webservices depends on the size, pattern and the nature of the real life dataset accesible from these services. Hence creating a dummy dataset might not actually reflect the true performance of a webservice to be benchmarked. 2. Since many webservices have a non-naive backend which includes interaction with multiple backend components, tables and databases. Generating a dummy dataset for such webservices is a non-trivial and a time consuming task. However to benchmark a webservice before it has access to a real dataset, support for automatic data insertion can be added in the future. An example of such a scenario is benchmarking a webservice before it moves to production. ### 3. Run YCSB At this point we assume that you've setup a webservice accesible at an HTTP endpoint like this: http://{host}:{port}/{service_endpoint}. Before you are ready to run please ensure that you have prepared a trace for the CRUD operations to benchmark your webservice. Trace is a collection of URL resources that should be hit in order to benchmark any webservice. The more realistic this collection of URL is, the more reliable and accurate are the benchmarking results because this means simulating the real life workload more accurately. Tracefile is a file that holds the trace. For example, if your webservice exists at http://{host}:{port}/{endpoint}, and you want to benchmark the performance of READS on this webservice with five resources (namely resource_1, resource_2 ... resource_5) then the url.trace.read file will look like this: http://{host}:{port}/{endpoint}/resource_1 http://{host}:{port}/{endpoint}/resource_2 http://{host}:{port}/{endpoint}/resource_3 http://{host}:{port}/{endpoint}/resource_4 http://{host}:{port}/{endpoint}/resource_5 The rest module will pick up URLs from the above file according to the `requestdistribution` property (default is zipfian) mentioned in the rest_workload. In the example above we assume that the property `url.prefix` (see below for property description) is set to empty. If url.prefix property is set to `http://{host}:{port}/{endpoint}/` the equivalent of the read trace given above would look like: resource_1 resource_2 resource_3 resource_4 resource_5 In real life the traces for various CRUD operations are diffent from one another. HTTP GET will rarely have the same URL access pattern as that of HTTP POST or HTTP PUT. Hence to give enough flexibility to benchmark webservices, different trace files can be used for different CRUD operations. However if you wish to use the same trace for all these operations, just pass the same file to all these properties - `url.trace.read`, `url.trace.insert`, `url.trace.update` & `url.trace.delete`. Now you are ready to run! Run the rest_workload: ./bin/ycsb run rest -s -P workloads/rest_workload For further configuration see below: ### Default Configuration Parameters The default settings for the rest binding are as follows: - `url.prefix` - The base endpoint URL where the webservice is running. URLs from trace files (DELETE, GET, POST, PUT) will be prefixed with this value before making an HTTP request. A common usage value would be http://127.0.0.1:8080/{yourService} - Default value is `http://127.0.0.1:80/`. - `url.trace.read` - The path to a trace file that holds the URLs to be invoked for HTTP GET method. URLs must be seperated by a newline. - `url.trace.insert` - The path to a trace file that holds the URLs to be invoked for HTTP POST method. URLs must be seperated by a newline. - `url.trace.update` - The path to a trace file that holds the URLs to be invoked for HTTP PUT method. URLs must be seperated by a newline. - `url.trace.delete` - The path to a trace file that holds the URLs to be invoked for HTTP DELETE method. URLs must be seperated by a newline. - `headers` - The HTTP request headers used for all requests. Headers must be separated by space as a delimiter. - Default value is `Accept */* Accept-Language en-US,en;q=0.5 Content-Type application/x-www-form-urlencoded user-agent Mozilla/5.0` - `timeout.con` - The HTTP connection timeout in seconds. The response will be considered as an error if the client fails to connect with the server within this time limit. - Default value is `10` seconds. - `timeout.read` - The HTTP read timeout in seconds. The response will be considered as an error if the client fails to read from the server within this time limit. - Default value is `10` seconds. - `timeout.exec` - The time within which request must return a response. The response will be considered as an error if the client fails to complete the request within this time limit. - Default value is `10` seconds. - `log.enable` - A Boolean value to enable console status logs. When true, it will print all the HTTP requests being made and thier response status on the YCSB console window. - Default value is `false`. - `readrecordcount` - An integer value that signifies the top k URLs (entries) to be picked from the `url.trace.read` file for making HTTP GET requests. Must have a value greater than 0. If this value exceeds the number of entries present in `url.trace.read` file, then k will be set to the number of entries in the file. - Default value is `10000`. - `insertrecordcount` - An integer value that signifies the top k URLs to be picked from the `url.trace.insert` file for making HTTP POST requests. Must have a value greater than 0. If this value exceeds the number of entries present in `url.trace.insert` file, then k will be set to the number of entries in the file. - Default value is `5000`. - `deleterecordcount` - An integer value that signifies the top k URLs to be picked from the `url.trace.delete` file for making HTTP DELETE requests. Must have a value greater than 0. If this value exceeds the number of entries present in `url.trace.delete` file, then k will be set to the number of entries in the file. - Default value is `1000`. - `updaterecordcount` - An integer value that signifies the top k URLs to be picked from the `url.trace.update` file for making HTTP PUT requests. Must have a value greater than 0. If this value exceeds the number of entries present in `url.trace.update` file, then k will be set to the number of entries in the file. - Default value is `1000`. - `readzipfconstant` - An double value of the Zipf's constant to be used for insert requests. Applicable only if the requestdistribution = `zipfian`. - Default value is `0.9`. - `insertzipfconstant` - An double value of the Zipf's constant to be used for insert requests. Applicable only if the requestdistribution = `zipfian`. - Default value is `0.9`. - `updatezipfconstant` - An double value of the Zipf's constant to be used for insert requests. Applicable only if the requestdistribution = `zipfian`. - Default value is `0.9`. - `deletezipfconstant` - An double value of the Zipf's constant to be used for insert requests. Applicable only if the requestdistribution = `zipfian`. - Default value is `0.9`. ================================================ FILE: rest/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent rest-binding Rest Client Binding jar true 8.0.28 2.6 4.5.1 4.4.4 4.12 1.16.0 com.yahoo.ycsb core ${project.version} provided org.apache.httpcomponents httpclient ${httpclient.version} org.apache.httpcomponents httpcore ${httpcore.version} junit junit ${junit.version} test com.github.stefanbirkner system-rules ${system-rules.version} org.glassfish.jersey.core jersey-server ${jersey.version} org.glassfish.jersey.core jersey-client ${jersey.version} org.glassfish.jersey.containers jersey-container-servlet-core ${jersey.version} org.apache.tomcat tomcat-dbcp ${tomcat.version} org.apache.tomcat tomcat-juli org.apache.tomcat.embed tomcat-embed-core ${tomcat.version} org.apache.tomcat.embed tomcat-embed-logging-juli ${tomcat.version} org.apache.tomcat.embed tomcat-embed-logging-log4j ${tomcat.version} org.apache.tomcat.embed tomcat-embed-jasper ${tomcat.version} org.apache.tomcat.embed tomcat-embed-websocket ${tomcat.version} ================================================ FILE: rest/src/main/java/com/yahoo/ycsb/webservice/rest/RestClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.webservice.rest; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.util.zip.GZIPInputStream; import javax.ws.rs.HttpMethod; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; /** * Class responsible for making web service requests for benchmarking purpose. * Using Apache HttpClient over standard Java HTTP API as this is more flexible * and provides better functionality. For example HttpClient can automatically * handle redirects and proxy authentication which the standard Java API can't. */ public class RestClient extends DB { private static final String URL_PREFIX = "url.prefix"; private static final String CON_TIMEOUT = "timeout.con"; private static final String READ_TIMEOUT = "timeout.read"; private static final String EXEC_TIMEOUT = "timeout.exec"; private static final String LOG_ENABLED = "log.enable"; private static final String HEADERS = "headers"; private static final String COMPRESSED_RESPONSE = "response.compression"; private boolean compressedResponse; private boolean logEnabled; private String urlPrefix; private Properties props; private String[] headers; private CloseableHttpClient client; private int conTimeout = 10000; private int readTimeout = 10000; private int execTimeout = 10000; private volatile Criteria requestTimedout = new Criteria(false); @Override public void init() throws DBException { props = getProperties(); urlPrefix = props.getProperty(URL_PREFIX, "http://127.0.0.1:8080"); conTimeout = Integer.valueOf(props.getProperty(CON_TIMEOUT, "10")) * 1000; readTimeout = Integer.valueOf(props.getProperty(READ_TIMEOUT, "10")) * 1000; execTimeout = Integer.valueOf(props.getProperty(EXEC_TIMEOUT, "10")) * 1000; logEnabled = Boolean.valueOf(props.getProperty(LOG_ENABLED, "false").trim()); compressedResponse = Boolean.valueOf(props.getProperty(COMPRESSED_RESPONSE, "false").trim()); headers = props.getProperty(HEADERS, "Accept */* Content-Type application/xml user-agent Mozilla/5.0 ").trim() .split(" "); setupClient(); } private void setupClient() { RequestConfig.Builder requestBuilder = RequestConfig.custom(); requestBuilder = requestBuilder.setConnectTimeout(conTimeout); requestBuilder = requestBuilder.setConnectionRequestTimeout(readTimeout); requestBuilder = requestBuilder.setSocketTimeout(readTimeout); HttpClientBuilder clientBuilder = HttpClientBuilder.create().setDefaultRequestConfig(requestBuilder.build()); this.client = clientBuilder.setConnectionManagerShared(true).build(); } @Override public Status read(String table, String endpoint, Set fields, Map result) { int responseCode; try { responseCode = httpGet(urlPrefix + endpoint, result); } catch (Exception e) { responseCode = handleExceptions(e, urlPrefix + endpoint, HttpMethod.GET); } if (logEnabled) { System.err.println(new StringBuilder("GET Request: ").append(urlPrefix).append(endpoint) .append(" | Response Code: ").append(responseCode).toString()); } return getStatus(responseCode); } @Override public Status insert(String table, String endpoint, Map values) { int responseCode; try { responseCode = httpExecute(new HttpPost(urlPrefix + endpoint), values.get("data").toString()); } catch (Exception e) { responseCode = handleExceptions(e, urlPrefix + endpoint, HttpMethod.POST); } if (logEnabled) { System.err.println(new StringBuilder("POST Request: ").append(urlPrefix).append(endpoint) .append(" | Response Code: ").append(responseCode).toString()); } return getStatus(responseCode); } @Override public Status delete(String table, String endpoint) { int responseCode; try { responseCode = httpDelete(urlPrefix + endpoint); } catch (Exception e) { responseCode = handleExceptions(e, urlPrefix + endpoint, HttpMethod.DELETE); } if (logEnabled) { System.err.println(new StringBuilder("DELETE Request: ").append(urlPrefix).append(endpoint) .append(" | Response Code: ").append(responseCode).toString()); } return getStatus(responseCode); } @Override public Status update(String table, String endpoint, Map values) { int responseCode; try { responseCode = httpExecute(new HttpPut(urlPrefix + endpoint), values.get("data").toString()); } catch (Exception e) { responseCode = handleExceptions(e, urlPrefix + endpoint, HttpMethod.PUT); } if (logEnabled) { System.err.println(new StringBuilder("PUT Request: ").append(urlPrefix).append(endpoint) .append(" | Response Code: ").append(responseCode).toString()); } return getStatus(responseCode); } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { return Status.NOT_IMPLEMENTED; } // Maps HTTP status codes to YCSB status codes. private Status getStatus(int responseCode) { int rc = responseCode / 100; if (responseCode == 400) { return Status.BAD_REQUEST; } else if (responseCode == 403) { return Status.FORBIDDEN; } else if (responseCode == 404) { return Status.NOT_FOUND; } else if (responseCode == 501) { return Status.NOT_IMPLEMENTED; } else if (responseCode == 503) { return Status.SERVICE_UNAVAILABLE; } else if (rc == 5) { return Status.ERROR; } return Status.OK; } private int handleExceptions(Exception e, String url, String method) { if (logEnabled) { System.err.println(new StringBuilder(method).append(" Request: ").append(url).append(" | ") .append(e.getClass().getName()).append(" occured | Error message: ") .append(e.getMessage()).toString()); } if (e instanceof ClientProtocolException) { return 400; } return 500; } // Connection is automatically released back in case of an exception. private int httpGet(String endpoint, Map result) throws IOException { requestTimedout.setIsSatisfied(false); Thread timer = new Thread(new Timer(execTimeout, requestTimedout)); timer.start(); int responseCode = 200; HttpGet request = new HttpGet(endpoint); for (int i = 0; i < headers.length; i = i + 2) { request.setHeader(headers[i], headers[i + 1]); } CloseableHttpResponse response = client.execute(request); responseCode = response.getStatusLine().getStatusCode(); HttpEntity responseEntity = response.getEntity(); // If null entity don't bother about connection release. if (responseEntity != null) { InputStream stream = responseEntity.getContent(); /* * TODO: Gzip Compression must be supported in the future. Header[] * header = response.getAllHeaders(); * if(response.getHeaders("Content-Encoding")[0].getValue().contains * ("gzip")) stream = new GZIPInputStream(stream); */ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8")); StringBuffer responseContent = new StringBuffer(); String line = ""; while ((line = reader.readLine()) != null) { if (requestTimedout.isSatisfied()) { // Must avoid memory leak. reader.close(); stream.close(); EntityUtils.consumeQuietly(responseEntity); response.close(); client.close(); throw new TimeoutException(); } responseContent.append(line); } timer.interrupt(); result.put("response", new StringByteIterator(responseContent.toString())); // Closing the input stream will trigger connection release. stream.close(); } EntityUtils.consumeQuietly(responseEntity); response.close(); client.close(); return responseCode; } private int httpExecute(HttpEntityEnclosingRequestBase request, String data) throws IOException { requestTimedout.setIsSatisfied(false); Thread timer = new Thread(new Timer(execTimeout, requestTimedout)); timer.start(); int responseCode = 200; for (int i = 0; i < headers.length; i = i + 2) { request.setHeader(headers[i], headers[i + 1]); } InputStreamEntity reqEntity = new InputStreamEntity(new ByteArrayInputStream(data.getBytes()), ContentType.APPLICATION_FORM_URLENCODED); reqEntity.setChunked(true); request.setEntity(reqEntity); CloseableHttpResponse response = client.execute(request); responseCode = response.getStatusLine().getStatusCode(); HttpEntity responseEntity = response.getEntity(); // If null entity don't bother about connection release. if (responseEntity != null) { InputStream stream = responseEntity.getContent(); if (compressedResponse) { stream = new GZIPInputStream(stream); } BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8")); StringBuffer responseContent = new StringBuffer(); String line = ""; while ((line = reader.readLine()) != null) { if (requestTimedout.isSatisfied()) { // Must avoid memory leak. reader.close(); stream.close(); EntityUtils.consumeQuietly(responseEntity); response.close(); client.close(); throw new TimeoutException(); } responseContent.append(line); } timer.interrupt(); // Closing the input stream will trigger connection release. stream.close(); } EntityUtils.consumeQuietly(responseEntity); response.close(); client.close(); return responseCode; } private int httpDelete(String endpoint) throws IOException { requestTimedout.setIsSatisfied(false); Thread timer = new Thread(new Timer(execTimeout, requestTimedout)); timer.start(); int responseCode = 200; HttpDelete request = new HttpDelete(endpoint); for (int i = 0; i < headers.length; i = i + 2) { request.setHeader(headers[i], headers[i + 1]); } CloseableHttpResponse response = client.execute(request); responseCode = response.getStatusLine().getStatusCode(); response.close(); client.close(); return responseCode; } /** * Marks the input {@link Criteria} as satisfied when the input time has elapsed. */ class Timer implements Runnable { private long timeout; private Criteria timedout; public Timer(long timeout, Criteria timedout) { this.timedout = timedout; this.timeout = timeout; } @Override public void run() { try { Thread.sleep(timeout); this.timedout.setIsSatisfied(true); } catch (InterruptedException e) { // Do nothing. } } } /** * Sets the flag when a criteria is fulfilled. */ class Criteria { private boolean isSatisfied; public Criteria(boolean isSatisfied) { this.isSatisfied = isSatisfied; } public boolean isSatisfied() { return isSatisfied; } public void setIsSatisfied(boolean satisfied) { this.isSatisfied = satisfied; } } /** * Private exception class for execution timeout. */ class TimeoutException extends RuntimeException { private static final long serialVersionUID = 1L; public TimeoutException() { super("HTTP Request exceeded execution time limit."); } } } ================================================ FILE: rest/src/main/java/com/yahoo/ycsb/webservice/rest/package-info.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for RESTFul Web Services. */ package com.yahoo.ycsb.webservice.rest; ================================================ FILE: rest/src/test/java/com/yahoo/ycsb/webservice/rest/IntegrationTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.webservice.rest; import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.List; import javax.servlet.ServletException; import org.apache.catalina.Context; import org.apache.catalina.LifecycleException; import org.apache.catalina.startup.Tomcat; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletContainer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.FixMethodOrder; import org.junit.Rule; import org.junit.Test; import org.junit.contrib.java.lang.system.Assertion; import org.junit.contrib.java.lang.system.ExpectedSystemExit; import org.junit.runners.MethodSorters; import com.yahoo.ycsb.Client; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.webservice.rest.Utils; /** * Integration test cases to verify the end to end working of the rest-binding * module. It performs these steps in order. 1. Runs an embedded Tomcat * server with a mock RESTFul web service. 2. Invokes the {@link Client} * class with the required parameters to start benchmarking the mock REST * service. 3. Compares the response stored in the output file by {@link Client} * class with the response expected. 4. Stops the embedded Tomcat server. * Cases for verifying the handling of different HTTP status like 2xx & 5xx have * been included in success and failure test cases. */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class IntegrationTest { @Rule public final ExpectedSystemExit exit = ExpectedSystemExit.none(); private static int port = 8080; private static Tomcat tomcat; private static final String WORKLOAD_FILEPATH = IntegrationTest.class.getClassLoader().getResource("workload_rest").getPath(); private static final String TRACE_FILEPATH = IntegrationTest.class.getClassLoader().getResource("trace.txt").getPath(); private static final String ERROR_TRACE_FILEPATH = IntegrationTest.class.getClassLoader().getResource("error_trace.txt").getPath(); private static final String RESULTS_FILEPATH = IntegrationTest.class.getClassLoader().getResource(".").getPath() + "results.txt"; @BeforeClass public static void init() throws ServletException, LifecycleException, FileNotFoundException, IOException, DBException, InterruptedException { String webappDirLocation = IntegrationTest.class.getClassLoader().getResource("WebContent").getPath(); while (!Utils.available(port)) { port++; } tomcat = new Tomcat(); tomcat.setPort(Integer.valueOf(port)); Context context = tomcat.addWebapp("/webService", new File(webappDirLocation).getAbsolutePath()); Tomcat.addServlet(context, "jersey-container-servlet", resourceConfig()); context.addServletMapping("/rest/*", "jersey-container-servlet"); tomcat.start(); // Allow time for proper startup. Thread.sleep(1000); } @AfterClass public static void cleanUp() throws LifecycleException { tomcat.stop(); } // All read operations during benchmark are executed successfully with an HTTP OK status. @Test public void testReadOpsBenchmarkSuccess() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[READ], Return=OK, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(TRACE_FILEPATH, 1, 0, 0, 0)); } //All read operations during benchmark are executed with an HTTP 500 error. @Test public void testReadOpsBenchmarkFailure() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[READ], Return=ERROR, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(ERROR_TRACE_FILEPATH, 1, 0, 0, 0)); } //All insert operations during benchmark are executed successfully with an HTTP OK status. @Test public void testInsertOpsBenchmarkSuccess() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[INSERT], Return=OK, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(TRACE_FILEPATH, 0, 1, 0, 0)); } //All read operations during benchmark are executed with an HTTP 500 error. @Test public void testInsertOpsBenchmarkFailure() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[INSERT], Return=ERROR, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(ERROR_TRACE_FILEPATH, 0, 1, 0, 0)); } //All update operations during benchmark are executed successfully with an HTTP OK status. @Test public void testUpdateOpsBenchmarkSuccess() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[UPDATE], Return=OK, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(TRACE_FILEPATH, 0, 0, 1, 0)); } //All read operations during benchmark are executed with an HTTP 500 error. @Test public void testUpdateOpsBenchmarkFailure() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[UPDATE], Return=ERROR, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(ERROR_TRACE_FILEPATH, 0, 0, 1, 0)); } //All delete operations during benchmark are executed successfully with an HTTP OK status. @Test public void testDeleteOpsBenchmarkSuccess() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[DELETE], Return=OK, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(TRACE_FILEPATH, 0, 0, 0, 1)); } //All read operations during benchmark are executed with an HTTP 500 error. @Test public void testDeleteOpsBenchmarkFailure() throws InterruptedException { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { List results = Utils.read(RESULTS_FILEPATH); assertEquals(true, results.contains("[DELETE], Return=ERROR, 1")); Utils.delete(RESULTS_FILEPATH); } }); Client.main(getArgs(ERROR_TRACE_FILEPATH, 0, 0, 0, 1)); } private String[] getArgs(String traceFilePath, float rp, float ip, float up, float dp) { String[] args = new String[25]; args[0] = "-target"; args[1] = "1"; args[2] = "-t"; args[3] = "-P"; args[4] = WORKLOAD_FILEPATH; args[5] = "-p"; args[6] = "url.prefix=http://127.0.0.1:"+port+"/webService/rest/resource/"; args[7] = "-p"; args[8] = "url.trace.read=" + traceFilePath; args[9] = "-p"; args[10] = "url.trace.insert=" + traceFilePath; args[11] = "-p"; args[12] = "url.trace.update=" + traceFilePath; args[13] = "-p"; args[14] = "url.trace.delete=" + traceFilePath; args[15] = "-p"; args[16] = "exportfile=" + RESULTS_FILEPATH; args[17] = "-p"; args[18] = "readproportion=" + rp; args[19] = "-p"; args[20] = "updateproportion=" + up; args[21] = "-p"; args[22] = "deleteproportion=" + dp; args[23] = "-p"; args[24] = "insertproportion=" + ip; return args; } private static ServletContainer resourceConfig() { return new ServletContainer(new ResourceConfig(new ResourceLoader().getClasses())); } } ================================================ FILE: rest/src/test/java/com/yahoo/ycsb/webservice/rest/ResourceLoader.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.webservice.rest; import java.util.HashSet; import java.util.Set; import javax.ws.rs.core.Application; /** * Class responsible for loading mock rest resource class like * {@link RestTestResource}. */ public class ResourceLoader extends Application { @Override public Set> getClasses() { final Set> classes = new HashSet>(); classes.add(RestTestResource.class); return classes; } } ================================================ FILE: rest/src/test/java/com/yahoo/ycsb/webservice/rest/RestClientTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.webservice.rest; import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.HashMap; import java.util.Properties; import javax.servlet.ServletException; import org.apache.catalina.Context; import org.apache.catalina.LifecycleException; import org.apache.catalina.startup.Tomcat; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletContainer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; /** * Test cases to verify the {@link RestClient} of the rest-binding * module. It performs these steps in order. 1. Runs an embedded Tomcat * server with a mock RESTFul web service. 2. Invokes the {@link RestClient} * class for all the various methods which make HTTP calls to the mock REST * service. 3. Compares the response from such calls to the mock REST * service with the response expected. 4. Stops the embedded Tomcat server. * Cases for verifying the handling of different HTTP status like 2xx, 4xx & * 5xx have been included in success and failure test cases. */ public class RestClientTest { private static Integer port = 8080; private static Tomcat tomcat; private static RestClient rc = new RestClient(); private static final String RESPONSE_TAG = "response"; private static final String DATA_TAG = "data"; private static final String VALID_RESOURCE = "resource_valid"; private static final String INVALID_RESOURCE = "resource_invalid"; private static final String ABSENT_RESOURCE = "resource_absent"; private static final String UNAUTHORIZED_RESOURCE = "resource_unauthorized"; private static final String INPUT_DATA = "onetwo"; @BeforeClass public static void init() throws IOException, DBException, ServletException, LifecycleException, InterruptedException { String webappDirLocation = IntegrationTest.class.getClassLoader().getResource("WebContent").getPath(); while (!Utils.available(port)) { port++; } tomcat = new Tomcat(); tomcat.setPort(Integer.valueOf(port)); Context context = tomcat.addWebapp("/webService", new File(webappDirLocation).getAbsolutePath()); Tomcat.addServlet(context, "jersey-container-servlet", resourceConfig()); context.addServletMapping("/rest/*", "jersey-container-servlet"); tomcat.start(); // Allow time for proper startup. Thread.sleep(1000); Properties props = new Properties(); props.load(new FileReader(RestClientTest.class.getClassLoader().getResource("workload_rest").getPath())); // Update the port value in the url.prefix property. props.setProperty("url.prefix", props.getProperty("url.prefix").replaceAll("PORT", port.toString())); rc.setProperties(props); rc.init(); } @AfterClass public static void cleanUp() throws DBException { rc.cleanup(); } // Read success. @Test public void read_200() { HashMap result = new HashMap(); Status status = rc.read(null, VALID_RESOURCE, null, result); assertEquals(Status.OK, status); assertEquals(result.get(RESPONSE_TAG).toString(), "HTTP GET response to: "+ VALID_RESOURCE); } // Unauthorized request error. @Test public void read_403() { HashMap result = new HashMap(); Status status = rc.read(null, UNAUTHORIZED_RESOURCE, null, result); assertEquals(Status.FORBIDDEN, status); } //Not found error. @Test public void read_404() { HashMap result = new HashMap(); Status status = rc.read(null, ABSENT_RESOURCE, null, result); assertEquals(Status.NOT_FOUND, status); } // Server error. @Test public void read_500() { HashMap result = new HashMap(); Status status = rc.read(null, INVALID_RESOURCE, null, result); assertEquals(Status.ERROR, status); } // Insert success. @Test public void insert_200() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.insert(null, VALID_RESOURCE, data); assertEquals(Status.OK, status); } @Test public void insert_403() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.insert(null, UNAUTHORIZED_RESOURCE, data); assertEquals(Status.FORBIDDEN, status); } @Test public void insert_404() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.insert(null, ABSENT_RESOURCE, data); assertEquals(Status.NOT_FOUND, status); } @Test public void insert_500() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.insert(null, INVALID_RESOURCE, data); assertEquals(Status.ERROR, status); } // Delete success. @Test public void delete_200() { Status status = rc.delete(null, VALID_RESOURCE); assertEquals(Status.OK, status); } @Test public void delete_403() { Status status = rc.delete(null, UNAUTHORIZED_RESOURCE); assertEquals(Status.FORBIDDEN, status); } @Test public void delete_404() { Status status = rc.delete(null, ABSENT_RESOURCE); assertEquals(Status.NOT_FOUND, status); } @Test public void delete_500() { Status status = rc.delete(null, INVALID_RESOURCE); assertEquals(Status.ERROR, status); } @Test public void update_200() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.update(null, VALID_RESOURCE, data); assertEquals(Status.OK, status); } @Test public void update_403() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.update(null, UNAUTHORIZED_RESOURCE, data); assertEquals(Status.FORBIDDEN, status); } @Test public void update_404() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.update(null, ABSENT_RESOURCE, data); assertEquals(Status.NOT_FOUND, status); } @Test public void update_500() { HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); Status status = rc.update(null, INVALID_RESOURCE, data); assertEquals(Status.ERROR, status); } @Test public void scan() { assertEquals(Status.NOT_IMPLEMENTED, rc.scan(null, null, 0, null, null)); } private static ServletContainer resourceConfig() { return new ServletContainer(new ResourceConfig(new ResourceLoader().getClasses())); } } ================================================ FILE: rest/src/test/java/com/yahoo/ycsb/webservice/rest/RestTestResource.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.webservice.rest; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.HttpMethod; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; /** * Class that implements a mock RESTFul web service to be used for integration * testing. */ @Path("/resource/{id}") public class RestTestResource { @GET @Produces(MediaType.TEXT_PLAIN) public Response respondToGET(@PathParam("id") String id) { return processRequests(id, HttpMethod.GET); } @POST @Produces(MediaType.TEXT_PLAIN) public Response respondToPOST(@PathParam("id") String id) { return processRequests(id, HttpMethod.POST); } @DELETE @Produces(MediaType.TEXT_PLAIN) public Response respondToDELETE(@PathParam("id") String id) { return processRequests(id, HttpMethod.DELETE); } @PUT @Produces(MediaType.TEXT_PLAIN) public Response respondToPUT(@PathParam("id") String id) { return processRequests(id, HttpMethod.PUT); } private static Response processRequests(String id, String method) { if (id.equals("resource_invalid")) return Response.serverError().build(); else if (id.equals("resource_absent")) return Response.status(Response.Status.NOT_FOUND).build(); else if (id.equals("resource_unauthorized")) return Response.status(Response.Status.FORBIDDEN).build(); return Response.ok("HTTP " + method + " response to: " + id).build(); } } ================================================ FILE: rest/src/test/java/com/yahoo/ycsb/webservice/rest/Utils.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.webservice.rest; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.net.DatagramSocket; import java.net.ServerSocket; import java.util.ArrayList; import java.util.List; /** * Holds the common utility methods. */ public class Utils { /** * Returns true if the port is available. * * @param port * @return isAvailable */ public static boolean available(int port) { ServerSocket ss = null; DatagramSocket ds = null; try { ss = new ServerSocket(port); ss.setReuseAddress(true); ds = new DatagramSocket(port); ds.setReuseAddress(true); return true; } catch (IOException e) { } finally { if (ds != null) { ds.close(); } if (ss != null) { try { ss.close(); } catch (IOException e) { /* should not be thrown */ } } } return false; } public static List read(String filepath) { List list = new ArrayList(); try { BufferedReader file = new BufferedReader(new FileReader(filepath)); String line = null; while ((line = file.readLine()) != null) { list.add(line.trim()); } file.close(); } catch (IOException e) { e.printStackTrace(); } return list; } public static void delete(String filepath) { try { new File(filepath).delete(); } catch (Exception e) { e.printStackTrace(); } } } ================================================ FILE: rest/src/test/resources/WebContent/index.html ================================================ rest-binding Welcome to the rest-binding integration test cases! ================================================ FILE: rest/src/test/resources/error_trace.txt ================================================ resource_invalid ================================================ FILE: rest/src/test/resources/trace.txt ================================================ resource_1 resource_2 resource_3 resource_4 resource_5 ================================================ FILE: rest/src/test/resources/workload_rest ================================================ # Copyright (c) 2016 Yahoo! Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload A: Update heavy workload # Application example: Session store recording recent actions # # Read/update ratio: 50/50 # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) # Request distribution: zipfian # Core Properties workload=com.yahoo.ycsb.workloads.RestWorkload db=com.yahoo.ycsb.webservice.rest.RestClient exporter=com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter threadcount=1 fieldlengthdistribution=uniform measurementtype=hdrhistogram # Workload Properties fieldcount=1 fieldlength=2500 readproportion=1 updateproportion=0 deleteproportion=0 insertproportion=0 requestdistribution=zipfian operationcount=1 maxexecutiontime=720 # Custom Properties url.prefix=http://127.0.0.1:PORT/webService/rest/resource/ url.trace.read=/src/test/resource/trace.txt url.trace.insert=/src/test/resource/trace.txt url.trace.update=/src/test/resource/trace.txt url.trace.delete=/src/test/resource/trace.txt # Header must be separated by space. Other delimiters might occur as header values and hence can not be used. headers=Accept */* Accept-Language en-US,en;q=0.5 Content-Type application/x-www-form-urlencoded user-agent Mozilla/5.0 Connection close timeout.con=60 timeout.read=60 timeout.exec=60 log.enable=false readrecordcount=10000 insertrecordcount=5000 deleterecordcount=1000 updaterecordcount=1000 readzipfconstant=0.9 insertzipfconstant=0.9 updatezipfconstant=0.9 deletezipfconstant=0.9 # Measurement Properties hdrhistogram.percentiles=50,90,95,99 histogram.buckets=1 ================================================ FILE: riak/README.md ================================================ Riak KV Client for Yahoo! Cloud System Benchmark (YCSB) ======================================================= The Riak KV YCSB client is designed to work with the Yahoo! Cloud System Benchmark (YCSB) project (https://github.com/brianfrankcooper/YCSB) to support performance testing for the 2.x.y line of the Riak KV database. Creating a bucket-type to use with YCSB ---------------------------- Perform the following operations on your Riak cluster to configure it for the benchmarks. Set the default backend for Riak to LevelDB in the `riak.conf` file of every node of your cluster. This is required to support secondary indexes, which are used for the `scan` transactions. You can do this by modifying the proper line as shown below. ``` storage_backend = leveldb ``` After this, create a bucket type named "ycsb"[1](#f1) by logging into one of the nodes in your cluster. Now you're ready to set up the cluster to operate using one between strong and eventual consistency model as shown in the next two subsections. ###Strong consistency model To use the strong consistency model (default), you need to follow the next two steps. 1. In every `riak.conf` file, search for the `##strong_consistency=on` line and uncomment it. It's important that you do this before you start your cluster! 2. Run the following `riak-admin` commands: ``` riak-admin bucket-type create ycsb '{"props":{"consistent":true}}' riak-admin bucket-type activate ycsb ``` When using this model, you **may want to specify the number of replicas to create for each object**[2](#f2): the *R* and *W* parameters (see next section) will in fact be ignored. The only information needed by this consistency model is how many nodes the system has to successfully query to consider a transaction completed. To set this parameter, you can add `"n_val":N` to the list of properties shown above (by default `N` is set to 3). ####A note on the scan transactions Currently, `scan` transactions are not _directly_ supported, as there is no suitable mean to perform them properly. This will not cause the benchmark to fail, it simply won't perform any scan transaction at all (these will immediately return with a `Status.NOT_IMPLEMENTED` code). However, a possible workaround has been provided: considering that Riak doesn't allow strong-consistent bucket-types to use secondary indexes, we can create an eventually consistent one just to store (*key*, *2i indexes*) pairs. This will be later used only to obtain the keys where the objects are located, which will be then used to retrieve the actual objects from the strong-consistent bucket. If you want to use this workaround, then you have to create and activate a "_fake bucket-type_" using the following commands: ``` riak-admin bucket-type create fakeBucketType '{"props":{"allow_mult":"false","n_val":1,"dvv_enabled":false,"last_write_wins":true}}' riak-admin bucket-type activate fakeBucketType ``` A bucket-type so defined isn't allowed to _create siblings_ (`allow_mult":"false"`), it'll have just _one replica_ (`"n_val":1`) which'll store the _last value provided_ (`"last_write_wins":true`) and _vector clocks_ will be used instead of _dotted version vectors_ (`"dvv_enabled":false`). Note that setting `"n_val":1` means that the `scan` transactions won't be much *fault-tolerant*, considering that if a node fails then a lot of them could potentially fail. You may indeed increase this value, but this choice will necessarily load the cluster with more work. So, the choice is yours to make! Then you have to set the `riak.strong_consistent_scans_bucket_type` property (see next section) equal to the name you gave to the aforementioned "fake bucket-type" (e.g. `fakeBucketType` in this case). Please note that this workaround involves a **double store operation for each insert transaction**, one to store the actual object and another one to save the corresponding 2i index. In practice, the client won't notice any difference, as the latter operation is performed asynchronously. However, the cluster will be obviously loaded more, and this is why the proposed "fake bucket-type" to create is as less _resource-demanding_ as possible. ###Eventual consistency model If you want to use the eventual consistency model implemented in Riak, you have just to type: ``` riak-admin bucket-type create ycsb '{"props":{"allow_mult":"false"}}' riak-admin bucket-type activate ycsb ``` Riak KV configuration parameters ---------------------------- You can either specify these configuration parameters via command line or set them in the `riak.properties` file. * `riak.hosts` - string list, comma separated list of IPs or FQDNs. For example: `riak.hosts=127.0.0.1,127.0.0.2,127.0.0.3` or `riak.hosts=riak1.mydomain.com,riak2.mydomain.com,riak3.mydomain.com`. * `riak.port` - int, the port on which every node is listening. It must match the one specified in the `riak.conf` file at the line `listener.protobuf.internal`. * `riak.bucket_type` - string, it must match the name of the bucket type created during setup (see section above). * `riak.r_val` - int, this value represents the number of Riak nodes that must return results for a read operation before the transaction is considered successfully completed. * `riak.w_val` - int, this value represents the number of Riak nodes that must report success before an insert/update transaction is considered complete. * `riak.read_retry_count` - int, the number of times the client will try to read a key from Riak. * `riak.wait_time_before_retry` - int, the time (in milliseconds) before the client attempts to perform another read if the previous one failed. * `riak.transaction_time_limit` - int, the time (in seconds) the client waits before aborting the current transaction. * `riak.strong_consistency` - boolean, indicates whether to use *strong consistency* (true) or *eventual consistency* (false). * `riak.strong_consistent_scans_bucket_type` - **string**, indicates the bucket-type to use to allow scans transactions when using strong consistency mode. * `riak.debug` - boolean, enables debug mode. This displays all the properties (specified or defaults) when a benchmark is started. Moreover, it shows error causes whenever these occur. Note: For more information on workloads and how to run them please see: https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload 1 As specified in the `riak.properties` file. See parameters configuration section for further info. [↩](#a1) 2 More info about properly setting up a fault-tolerant cluster can be found at http://docs.basho.com/riak/kv/2.1.4/configuring/strong-consistency/#enabling-strong-consistency.[↩](#a2) ================================================ FILE: riak/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent riak-binding Riak KV Binding jar com.basho.riak riak-client 2.0.5 com.yahoo.ycsb core ${project.version} provided com.google.collections google-collections 1.0 junit junit 4.12 test ================================================ FILE: riak/src/main/java/com/yahoo/ycsb/db/riak/RiakKVClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors All rights reserved. * Copyright 2014 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.riak; import com.basho.riak.client.api.commands.buckets.StoreBucketProperties; import com.basho.riak.client.api.commands.kv.StoreValue; import com.basho.riak.client.api.commands.kv.UpdateValue; import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.query.RiakObject; import com.basho.riak.client.core.query.indexes.LongIntIndex; import com.basho.riak.client.core.util.BinaryValue; import com.yahoo.ycsb.*; import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import com.basho.riak.client.api.RiakClient; import com.basho.riak.client.api.cap.Quorum; import com.basho.riak.client.api.commands.indexes.IntIndexQuery; import com.basho.riak.client.api.commands.kv.DeleteValue; import com.basho.riak.client.api.commands.kv.FetchValue; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakNode; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; import static com.yahoo.ycsb.db.riak.RiakUtils.createResultHashMap; import static com.yahoo.ycsb.db.riak.RiakUtils.getKeyAsLong; import static com.yahoo.ycsb.db.riak.RiakUtils.serializeTable; /** * Riak KV 2.x.y client for YCSB framework. * */ public class RiakKVClient extends DB { private static final String HOST_PROPERTY = "riak.hosts"; private static final String PORT_PROPERTY = "riak.port"; private static final String BUCKET_TYPE_PROPERTY = "riak.bucket_type"; private static final String R_VALUE_PROPERTY = "riak.r_val"; private static final String W_VALUE_PROPERTY = "riak.w_val"; private static final String READ_RETRY_COUNT_PROPERTY = "riak.read_retry_count"; private static final String WAIT_TIME_BEFORE_RETRY_PROPERTY = "riak.wait_time_before_retry"; private static final String TRANSACTION_TIME_LIMIT_PROPERTY = "riak.transaction_time_limit"; private static final String STRONG_CONSISTENCY_PROPERTY = "riak.strong_consistency"; private static final String STRONG_CONSISTENT_SCANS_BUCKET_TYPE_PROPERTY = "riak.strong_consistent_scans_bucket_type"; private static final String DEBUG_PROPERTY = "riak.debug"; private static final Status TIME_OUT = new Status("TIME_OUT", "Cluster didn't respond after maximum wait time."); private String[] hosts; private int port; private String bucketType; private String bucketType2i; private Quorum rvalue; private Quorum wvalue; private int readRetryCount; private int waitTimeBeforeRetry; private int transactionTimeLimit; private boolean strongConsistency; private String strongConsistentScansBucketType; private boolean performStrongConsistentScans; private boolean debug; private RiakClient riakClient; private RiakCluster riakCluster; private void loadDefaultProperties() { InputStream propFile = RiakKVClient.class.getClassLoader().getResourceAsStream("riak.properties"); Properties propsPF = new Properties(System.getProperties()); try { propsPF.load(propFile); } catch (IOException e) { e.printStackTrace(); } hosts = propsPF.getProperty(HOST_PROPERTY).split(","); port = Integer.parseInt(propsPF.getProperty(PORT_PROPERTY)); bucketType = propsPF.getProperty(BUCKET_TYPE_PROPERTY); rvalue = new Quorum(Integer.parseInt(propsPF.getProperty(R_VALUE_PROPERTY))); wvalue = new Quorum(Integer.parseInt(propsPF.getProperty(W_VALUE_PROPERTY))); readRetryCount = Integer.parseInt(propsPF.getProperty(READ_RETRY_COUNT_PROPERTY)); waitTimeBeforeRetry = Integer.parseInt(propsPF.getProperty(WAIT_TIME_BEFORE_RETRY_PROPERTY)); transactionTimeLimit = Integer.parseInt(propsPF.getProperty(TRANSACTION_TIME_LIMIT_PROPERTY)); strongConsistency = Boolean.parseBoolean(propsPF.getProperty(STRONG_CONSISTENCY_PROPERTY)); strongConsistentScansBucketType = propsPF.getProperty(STRONG_CONSISTENT_SCANS_BUCKET_TYPE_PROPERTY); debug = Boolean.parseBoolean(propsPF.getProperty(DEBUG_PROPERTY)); } private void loadProperties() { // First, load the default properties... loadDefaultProperties(); // ...then, check for some props set at command line! Properties props = getProperties(); String portString = props.getProperty(PORT_PROPERTY); if (portString != null) { port = Integer.parseInt(portString); } String hostsString = props.getProperty(HOST_PROPERTY); if (hostsString != null) { hosts = hostsString.split(","); } String bucketTypeString = props.getProperty(BUCKET_TYPE_PROPERTY); if (bucketTypeString != null) { bucketType = bucketTypeString; } String rValueString = props.getProperty(R_VALUE_PROPERTY); if (rValueString != null) { rvalue = new Quorum(Integer.parseInt(rValueString)); } String wValueString = props.getProperty(W_VALUE_PROPERTY); if (wValueString != null) { wvalue = new Quorum(Integer.parseInt(wValueString)); } String readRetryCountString = props.getProperty(READ_RETRY_COUNT_PROPERTY); if (readRetryCountString != null) { readRetryCount = Integer.parseInt(readRetryCountString); } String waitTimeBeforeRetryString = props.getProperty(WAIT_TIME_BEFORE_RETRY_PROPERTY); if (waitTimeBeforeRetryString != null) { waitTimeBeforeRetry = Integer.parseInt(waitTimeBeforeRetryString); } String transactionTimeLimitString = props.getProperty(TRANSACTION_TIME_LIMIT_PROPERTY); if (transactionTimeLimitString != null) { transactionTimeLimit = Integer.parseInt(transactionTimeLimitString); } String strongConsistencyString = props.getProperty(STRONG_CONSISTENCY_PROPERTY); if (strongConsistencyString != null) { strongConsistency = Boolean.parseBoolean(strongConsistencyString); } String strongConsistentScansBucketTypeString = props.getProperty(STRONG_CONSISTENT_SCANS_BUCKET_TYPE_PROPERTY); if (strongConsistentScansBucketTypeString != null) { strongConsistentScansBucketType = strongConsistentScansBucketTypeString; } String debugString = props.getProperty(DEBUG_PROPERTY); if (debugString != null) { debug = Boolean.parseBoolean(debugString); } } public void init() throws DBException { loadProperties(); RiakNode.Builder builder = new RiakNode.Builder().withRemotePort(port); List nodes = RiakNode.Builder.buildNodes(builder, Arrays.asList(hosts)); riakCluster = new RiakCluster.Builder(nodes).build(); try { riakCluster.start(); riakClient = new RiakClient(riakCluster); } catch (Exception e) { System.err.println("Unable to properly start up the cluster. Reason: " + e.toString()); throw new DBException(e); } // If strong consistency is in use, we need to change the bucket-type where the 2i indexes will be stored. if (strongConsistency && !strongConsistentScansBucketType.isEmpty()) { // The 2i indexes have to be stored in the appositely created strongConsistentScansBucketType: this however has // to be done only if the user actually created it! So, if the latter doesn't exist, then the scan transactions // will not be performed at all. bucketType2i = strongConsistentScansBucketType; performStrongConsistentScans = true; } else { // If instead eventual consistency is in use, then the 2i indexes have to be stored in the bucket-type // indicated with the bucketType variable. bucketType2i = bucketType; performStrongConsistentScans = false; } if (debug) { System.err.println("DEBUG ENABLED. Configuration parameters:"); System.err.println("-----------------------------------------"); System.err.println("Hosts: " + Arrays.toString(hosts)); System.err.println("Port: " + port); System.err.println("Bucket Type: " + bucketType); System.err.println("R Val: " + rvalue.toString()); System.err.println("W Val: " + wvalue.toString()); System.err.println("Read Retry Count: " + readRetryCount); System.err.println("Wait Time Before Retry: " + waitTimeBeforeRetry + " ms"); System.err.println("Transaction Time Limit: " + transactionTimeLimit + " s"); System.err.println("Consistency model: " + (strongConsistency ? "Strong" : "Eventual")); if (strongConsistency) { System.err.println("Strong Consistent Scan Transactions " + (performStrongConsistentScans ? "" : "NOT ") + "allowed."); } } } /** * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. * * @param table The name of the table (Riak bucket) * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ @Override public Status read(String table, String key, Set fields, Map result) { Location location = new Location(new Namespace(bucketType, table), key); FetchValue fv = new FetchValue.Builder(location).withOption(FetchValue.Option.R, rvalue).build(); FetchValue.Response response; try { response = fetch(fv); if (response.isNotFound()) { if (debug) { System.err.println("Unable to read key " + key + ". Reason: NOT FOUND"); } return Status.NOT_FOUND; } } catch (TimeoutException e) { if (debug) { System.err.println("Unable to read key " + key + ". Reason: TIME OUT"); } return TIME_OUT; } catch (Exception e) { if (debug) { System.err.println("Unable to read key " + key + ". Reason: " + e.toString()); } return Status.ERROR; } // Create the result HashMap. HashMap partialResult = new HashMap<>(); createResultHashMap(fields, response, partialResult); result.putAll(partialResult); return Status.OK; } /** * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in * a HashMap. * Note: The scan operation requires the use of secondary indexes (2i) and LevelDB. * * @param table The name of the table (Riak bucket) * @param startkey The record key of the first record to read. * @param recordcount The number of records to read * @param fields The list of fields to read, or null for all of them * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return Zero on success, a non-zero error code on error */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { if (strongConsistency && !performStrongConsistentScans) { return Status.NOT_IMPLEMENTED; } // The strong consistent bucket-type is not capable of storing 2i indexes. So, we need to read them from the fake // one (which we use only to store indexes). This is why, when using such a consistency model, the bucketType2i // variable is set to FAKE_BUCKET_TYPE. IntIndexQuery iiq = new IntIndexQuery .Builder(new Namespace(bucketType2i, table), "key", getKeyAsLong(startkey), Long.MAX_VALUE) .withMaxResults(recordcount) .withPaginationSort(true) .build(); Location location; RiakFuture future = riakClient.executeAsync(iiq); try { IntIndexQuery.Response response = future.get(transactionTimeLimit, TimeUnit.SECONDS); List entries = response.getEntries(); // If no entries were retrieved, then something bad happened... if (entries.size() == 0) { if (debug) { System.err.println("Unable to scan any record starting from key " + startkey + ", aborting transaction. " + "Reason: NOT FOUND"); } return Status.NOT_FOUND; } for (IntIndexQuery.Response.Entry entry : entries) { // If strong consistency is in use, then the actual location of the object we want to read is obtained by // fetching the key from the one retrieved with the 2i indexes search operation. if (strongConsistency) { location = new Location(new Namespace(bucketType, table), entry.getRiakObjectLocation().getKeyAsString()); } else { location = entry.getRiakObjectLocation(); } FetchValue fv = new FetchValue.Builder(location) .withOption(FetchValue.Option.R, rvalue) .build(); FetchValue.Response keyResponse = fetch(fv); if (keyResponse.isNotFound()) { if (debug) { System.err.println("Unable to scan all requested records starting from key " + startkey + ", aborting " + "transaction. Reason: NOT FOUND"); } return Status.NOT_FOUND; } // Create the partial result to add to the result vector. HashMap partialResult = new HashMap<>(); createResultHashMap(fields, keyResponse, partialResult); result.add(partialResult); } } catch (TimeoutException e) { if (debug) { System.err.println("Unable to scan all requested records starting from key " + startkey + ", aborting " + "transaction. Reason: TIME OUT"); } return TIME_OUT; } catch (Exception e) { if (debug) { System.err.println("Unable to scan all records starting from key " + startkey + ", aborting transaction. " + "Reason: " + e.toString()); } return Status.ERROR; } return Status.OK; } /** * Tries to perform a read and, whenever it fails, retries to do it. It actually does try as many time as indicated, * even if the function riakClient.execute(fv) throws an exception. This is needed for those situation in which the * cluster is unable to respond properly due to overload. Note however that if the cluster doesn't respond after * transactionTimeLimit, the transaction is discarded immediately. * * @param fv The value to fetch from the cluster. */ private FetchValue.Response fetch(FetchValue fv) throws TimeoutException { FetchValue.Response response = null; for (int i = 0; i < readRetryCount; i++) { RiakFuture future = riakClient.executeAsync(fv); try { response = future.get(transactionTimeLimit, TimeUnit.SECONDS); if (!response.isNotFound()) { break; } } catch (TimeoutException e) { // Let the callee decide how to handle this exception... throw new TimeoutException(); } catch (Exception e) { // Sleep for a few ms before retrying... try { Thread.sleep(waitTimeBeforeRetry); } catch (InterruptedException e1) { e1.printStackTrace(); } } } return response; } /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key. Also creates a secondary index (2i) for each record consisting of the key * converted to long to be used for the scan operation. * * @param table The name of the table (Riak bucket) * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error */ @Override public Status insert(String table, String key, Map values) { Location location = new Location(new Namespace(bucketType, table), key); RiakObject object = new RiakObject(); // Strong consistency doesn't support secondary indexing, but eventually consistent model does. So, we can mock a // 2i usage by creating a fake object stored in an eventually consistent bucket-type with the SAME KEY THAT THE // ACTUAL OBJECT HAS. This latter is obviously stored in the strong consistent bucket-type indicated with the // riak.bucket_type property. if (strongConsistency && performStrongConsistentScans) { // Create a fake object to store in the default bucket-type just to keep track of the 2i indices. Location fakeLocation = new Location(new Namespace(strongConsistentScansBucketType, table), key); // Obviously, we want the fake object to contain as less data as possible. We can't create a void object, so // we have to choose the minimum data size allowed: it is one byte. RiakObject fakeObject = new RiakObject(); fakeObject.setValue(BinaryValue.create(new byte[]{0x00})); fakeObject.getIndexes().getIndex(LongIntIndex.named("key_int")).add(getKeyAsLong(key)); StoreValue fakeStore = new StoreValue.Builder(fakeObject) .withLocation(fakeLocation) .build(); // We don't mind whether the operation is finished or not, because waiting for it to complete would slow down the // client and make our solution too heavy to be seen as a valid compromise. This will obviously mean that under // heavy load conditions a scan operation could fail due to an unfinished "fakeStore". riakClient.executeAsync(fakeStore); } else if (!strongConsistency) { // The next operation is useless when using strong consistency model, so it's ok to perform it only when using // eventual consistency. object.getIndexes().getIndex(LongIntIndex.named("key_int")).add(getKeyAsLong(key)); } // Store proper values into the object. object.setValue(BinaryValue.create(serializeTable(values))); StoreValue store = new StoreValue.Builder(object) .withOption(StoreValue.Option.W, wvalue) .withLocation(location) .build(); RiakFuture future = riakClient.executeAsync(store); try { future.get(transactionTimeLimit, TimeUnit.SECONDS); } catch (TimeoutException e) { if (debug) { System.err.println("Unable to insert key " + key + ". Reason: TIME OUT"); } return TIME_OUT; } catch (Exception e) { if (debug) { System.err.println("Unable to insert key " + key + ". Reason: " + e.toString()); } return Status.ERROR; } return Status.OK; } /** * Auxiliary class needed for object substitution within the update operation. It is a fundamental part of the * fetch-update (locally)-store cycle described by Basho to properly perform a strong-consistent update. */ private static final class UpdateEntity extends UpdateValue.Update { private final RiakObject object; private UpdateEntity(RiakObject object) { this.object = object; } //Simply returns the object. @Override public RiakObject apply(RiakObject original) { return object; } } /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the * record with the specified record key, overwriting any existing values with the same field name. * * @param table The name of the table (Riak bucket) * @param key The record key of the record to write. * @param values A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error */ @Override public Status update(String table, String key, Map values) { // If eventual consistency model is in use, then an update operation is pratically equivalent to an insert one. if (!strongConsistency) { return insert(table, key, values); } Location location = new Location(new Namespace(bucketType, table), key); UpdateValue update = new UpdateValue.Builder(location) .withUpdate(new UpdateEntity(new RiakObject().setValue(BinaryValue.create(serializeTable(values))))) .build(); RiakFuture future = riakClient.executeAsync(update); try { // For some reason, the update transaction doesn't throw any exception when no cluster has been started, so one // needs to check whether it was done or not. When calling the wasUpdated() function with no nodes available, a // NullPointerException is thrown. // Moreover, such exception could be thrown when more threads are trying to update the same key or, more // generally, when the system is being queried by many clients (i.e. overloaded). This is a known limitation of // Riak KV's strong consistency implementation. future.get(transactionTimeLimit, TimeUnit.SECONDS).wasUpdated(); } catch (TimeoutException e) { if (debug) { System.err.println("Unable to update key " + key + ". Reason: TIME OUT"); } return TIME_OUT; } catch (Exception e) { if (debug) { System.err.println("Unable to update key " + key + ". Reason: " + e.toString()); } return Status.ERROR; } return Status.OK; } /** * Delete a record from the database. * * @param table The name of the table (Riak bucket) * @param key The record key of the record to delete. * @return Zero on success, a non-zero error code on error */ @Override public Status delete(String table, String key) { Location location = new Location(new Namespace(bucketType, table), key); DeleteValue dv = new DeleteValue.Builder(location).build(); RiakFuture future = riakClient.executeAsync(dv); try { future.get(transactionTimeLimit, TimeUnit.SECONDS); } catch (TimeoutException e) { if (debug) { System.err.println("Unable to delete key " + key + ". Reason: TIME OUT"); } return TIME_OUT; } catch (Exception e) { if (debug) { System.err.println("Unable to delete key " + key + ". Reason: " + e.toString()); } return Status.ERROR; } return Status.OK; } public void cleanup() throws DBException { try { riakCluster.shutdown(); } catch (Exception e) { System.err.println("Unable to properly shutdown the cluster. Reason: " + e.toString()); throw new DBException(e); } } /** * Auxiliary function needed for testing. It configures the default bucket-type to take care of the consistency * problem by disallowing the siblings creation. Moreover, it disables strong consistency, because we don't have * the possibility to create a proper bucket-type to use to fake 2i indexes usage. * * @param bucket The bucket name. * @throws Exception Thrown if something bad happens. */ void setTestEnvironment(String bucket) throws Exception { bucketType = "default"; bucketType2i = bucketType; strongConsistency = false; Namespace ns = new Namespace(bucketType, bucket); StoreBucketProperties newBucketProperties = new StoreBucketProperties.Builder(ns).withAllowMulti(false).build(); riakClient.execute(newBucketProperties); } } ================================================ FILE: riak/src/main/java/com/yahoo/ycsb/db/riak/RiakUtils.java ================================================ /** * Copyright (c) 2016 YCSB contributors All rights reserved. * Copyright 2014 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.riak; import java.io.*; import java.util.HashMap; import java.util.Map; import java.util.Set; import com.basho.riak.client.api.commands.kv.FetchValue; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import static com.google.common.base.Preconditions.checkArgument; /** * Utility class for Riak KV Client. * */ final class RiakUtils { private RiakUtils() { super(); } private static byte[] toBytes(final int anInteger) { byte[] aResult = new byte[4]; aResult[0] = (byte) (anInteger >> 24); aResult[1] = (byte) (anInteger >> 16); aResult[2] = (byte) (anInteger >> 8); aResult[3] = (byte) (anInteger /* >> 0 */); return aResult; } private static int fromBytes(final byte[] aByteArray) { checkArgument(aByteArray.length == 4); return (aByteArray[0] << 24) | (aByteArray[1] & 0xFF) << 16 | (aByteArray[2] & 0xFF) << 8 | (aByteArray[3] & 0xFF); } private static void close(final OutputStream anOutputStream) { try { anOutputStream.close(); } catch (IOException e) { e.printStackTrace(); } } private static void close(final InputStream anInputStream) { try { anInputStream.close(); } catch (IOException e) { e.printStackTrace(); } } /** * Serializes a Map, transforming the contained list of (String, ByteIterator) couples into a byte array. * * @param aTable A Map to serialize. * @return A byte array containng the serialized table. */ static byte[] serializeTable(Map aTable) { final ByteArrayOutputStream anOutputStream = new ByteArrayOutputStream(); final Set> theEntries = aTable.entrySet(); try { for (final Map.Entry anEntry : theEntries) { final byte[] aColumnName = anEntry.getKey().getBytes(); anOutputStream.write(toBytes(aColumnName.length)); anOutputStream.write(aColumnName); final byte[] aColumnValue = anEntry.getValue().toArray(); anOutputStream.write(toBytes(aColumnValue.length)); anOutputStream.write(aColumnValue); } return anOutputStream.toByteArray(); } catch (IOException e) { throw new IllegalStateException(e); } finally { close(anOutputStream); } } /** * Deserializes an input byte array, transforming it into a list of (String, ByteIterator) pairs (i.e. a Map). * * @param aValue A byte array containing the table to deserialize. * @param theResult A Map containing the deserialized table. */ private static void deserializeTable(final byte[] aValue, final Map theResult) { final ByteArrayInputStream anInputStream = new ByteArrayInputStream(aValue); byte[] aSizeBuffer = new byte[4]; try { while (anInputStream.available() > 0) { anInputStream.read(aSizeBuffer); final int aColumnNameLength = fromBytes(aSizeBuffer); final byte[] aColumnNameBuffer = new byte[aColumnNameLength]; anInputStream.read(aColumnNameBuffer); anInputStream.read(aSizeBuffer); final int aColumnValueLength = fromBytes(aSizeBuffer); final byte[] aColumnValue = new byte[aColumnValueLength]; anInputStream.read(aColumnValue); theResult.put(new String(aColumnNameBuffer), new ByteArrayByteIterator(aColumnValue)); } } catch (Exception e) { throw new IllegalStateException(e); } finally { close(anInputStream); } } /** * Obtains a Long number from a key string. This will be the key used by Riak for all the transactions. * * @param key The key to convert from String to Long. * @return A Long number parsed from the key String. */ static Long getKeyAsLong(String key) { String keyString = key.replaceFirst("[a-zA-Z]*", ""); return Long.parseLong(keyString); } /** * Function that retrieves all the fields searched within a read or scan operation and puts them in the result * HashMap. * * @param fields The list of fields to read, or null for all of them. * @param response A Vector of HashMaps, where each HashMap is a set field/value pairs for one record. * @param resultHashMap The HashMap to return as result. */ static void createResultHashMap(Set fields, FetchValue.Response response, HashMapresultHashMap) { // If everything went fine, then a result must be given. Such an object is a hash table containing the (field, // value) pairs based on the requested fields. Note that in a read operation, ONLY ONE OBJECT IS RETRIEVED! // The following line retrieves the previously serialized table which was store with an insert transaction. byte[] responseFieldsAndValues = response.getValues().get(0).getValue().getValue(); // Deserialize the stored response table. HashMap deserializedTable = new HashMap<>(); deserializeTable(responseFieldsAndValues, deserializedTable); // If only specific fields are requested, then only these should be put in the result object! if (fields != null) { // Populate the HashMap to provide as result. for (Object field : fields.toArray()) { // Comparison between a requested field and the ones retrieved. If they're equal (i.e. the get() operation // DOES NOT return a null value), then proceed to store the pair in the resultHashMap. ByteIterator value = deserializedTable.get(field); if (value != null) { resultHashMap.put((String) field, value); } } } else { // If, instead, no field is specified, then all those retrieved must be provided as result. for (String field : deserializedTable.keySet()) { resultHashMap.put(field, deserializedTable.get(field)); } } } } ================================================ FILE: riak/src/main/java/com/yahoo/ycsb/db/riak/package-info.java ================================================ /** * Copyright (c) 2016 YCSB contributors All rights reserved. * Copyright 2014 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for Riak KV 2.x.y. * */ package com.yahoo.ycsb.db.riak; ================================================ FILE: riak/src/main/resources/riak.properties ================================================ ## # Copyright (c) 2016 YCSB contributors All rights reserved. # Copyright 2014 Basho Technologies, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # RiakKVClient - Default Properties # Note: Change the properties below to set the values to use for your test. You can set them either here or from the # command line. Note that the latter choice overrides these settings. # riak.hosts - string list, comma separated list of IPs or FQDNs. # EX: 127.0.0.1,127.0.0.2,127.0.0.3 or riak1.mydomain.com,riak2.mydomain.com,riak3.mydomain.com riak.hosts=127.0.0.1 # riak.port - int, the port on which every node is listening. It must match the one specified in the riak.conf file # at the line "listener.protobuf.internal". riak.port=8087 # riak.bucket_type - string, must match value of bucket type created during setup. See readme.md for more information riak.bucket_type=ycsb # riak.r_val - int, the R value represents the number of Riak nodes that must return results for a read before the read # is considered successful. riak.r_val=2 # riak.w_val - int, the W value represents the number of Riak nodes that must report success before an update is # considered complete. riak.w_val=2 # riak.read_retry_count - int, number of times the client will try to read a key from Riak. riak.read_retry_count=5 # riak.wait_time_before_retry - int, time (in milliseconds) the client waits before attempting to perform another # read if the previous one failed. riak.wait_time_before_retry=200 # riak.transaction_time_limit - int, time (in seconds) the client waits before aborting the current transaction. riak.transaction_time_limit=10 # riak.strong_consistency - boolean, indicates whether to use strong consistency (true) or eventual consistency (false). riak.strong_consistency=true # riak.strong_consistent_scans_bucket_type - string, indicates the bucket-type to use to allow scans transactions # when using strong consistency mode. Example: fakeBucketType. riak.strong_consistent_scans_bucket_type= # riak.debug - boolean, enables debug mode. This displays all the properties (specified or defaults) when a benchmark # is started. riak.debug=false ================================================ FILE: riak/src/test/java/com/yahoo/ycsb/db/riak/RiakKVClientTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors All rights reserved. * Copyright 2014 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.riak; import java.util.*; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assume.assumeNoException; import static org.junit.Assume.assumeThat; /** * Integration tests for the Riak KV client. */ public class RiakKVClientTest { private static RiakKVClient riakClient; private static final String bucket = "testBucket"; private static final String keyPrefix = "testKey"; private static final int recordsToInsert = 20; private static final int recordsToScan = 7; private static final String firstField = "Key number"; private static final String secondField = "Key number doubled"; private static final String thirdField = "Key number square"; private static boolean testStarted = false; /** * Creates a cluster for testing purposes. */ @BeforeClass public static void setUpClass() throws Exception { riakClient = new RiakKVClient(); riakClient.init(); // Set the test bucket environment with the appropriate parameters. try { riakClient.setTestEnvironment(bucket); } catch(Exception e) { assumeNoException("Unable to configure Riak KV for test, aborting.", e); } // Just add some records to work on... for (int i = 0; i < recordsToInsert; i++) { // Abort the entire test whenever the dataset population operation fails. assumeThat("Riak KV is NOT RUNNING, aborting test.", riakClient.insert(bucket, keyPrefix + String.valueOf(i), StringByteIterator.getByteIteratorMap( createExpectedHashMap(i))), is(Status.OK)); } // Variable to check to determine whether the test has started or not. testStarted = true; } /** * Shuts down the cluster created. */ @AfterClass public static void tearDownClass() throws Exception { // Delete all added keys before cleanup ONLY IF TEST ACTUALLY STARTED. if (testStarted) { for (int i = 0; i <= recordsToInsert; i++) { delete(keyPrefix + Integer.toString(i)); } } riakClient.cleanup(); } /** * Test method for read transaction. It is designed to read two of the three fields stored for each key, to also test * if the createResultHashMap() function implemented in RiakKVClient.java works as expected. */ @Test public void testRead() { // Choose a random key to read, among the available ones. int readKeyNumber = new Random().nextInt(recordsToInsert); // Prepare two fields to read. Set fields = new HashSet<>(); fields.add(firstField); fields.add(thirdField); // Prepare an expected result. HashMap expectedValue = new HashMap<>(); expectedValue.put(firstField, Integer.toString(readKeyNumber)); expectedValue.put(thirdField, Integer.toString(readKeyNumber * readKeyNumber)); // Define a HashMap to store the actual result. HashMap readValue = new HashMap<>(); // If a read transaction has been properly done, then one has to receive a Status.OK return from the read() // function. Moreover, the actual returned result MUST match the expected one. assertEquals("Read transaction FAILED.", Status.OK, riakClient.read(bucket, keyPrefix + Integer.toString(readKeyNumber), fields, readValue)); assertEquals("Read test FAILED. Actual read transaction value is NOT MATCHING the expected one.", expectedValue.toString(), readValue.toString()); } /** * Test method for scan transaction. A scan transaction has to be considered successfully completed only if all the * requested values are read (i.e. scan transaction returns with Status.OK). Moreover, one has to check if the * obtained results match the expected ones. */ @Test public void testScan() { // Choose, among the available ones, a random key as starting point for the scan transaction. int startScanKeyNumber = new Random().nextInt(recordsToInsert - recordsToScan); // Prepare a HashMap vector to store the scan transaction results. Vector> scannedValues = new Vector<>(); // Check whether the scan transaction is correctly performed or not. assertEquals("Scan transaction FAILED.", Status.OK, riakClient.scan(bucket, keyPrefix + Integer.toString(startScanKeyNumber), recordsToScan, null, scannedValues)); // After the scan transaction completes, compare the obtained results with the expected ones. for (int i = 0; i < recordsToScan; i++) { assertEquals("Scan test FAILED: the current scanned key is NOT MATCHING the expected one.", createExpectedHashMap(startScanKeyNumber + i).toString(), scannedValues.get(i).toString()); } } /** * Test method for update transaction. The test is designed to restore the previously read key. It is assumed to be * correct when, after performing the update transaction, one reads the just provided values. */ @Test public void testUpdate() { // Choose a random key to read, among the available ones. int updateKeyNumber = new Random().nextInt(recordsToInsert); // Define a HashMap to save the previously stored values for eventually restoring them. HashMap readValueBeforeUpdate = new HashMap<>(); riakClient.read(bucket, keyPrefix + Integer.toString(updateKeyNumber), null, readValueBeforeUpdate); // Prepare an update HashMap to store. HashMap updateValue = new HashMap<>(); updateValue.put(firstField, "UPDATED"); updateValue.put(secondField, "UPDATED"); updateValue.put(thirdField, "UPDATED"); // First of all, perform the update and check whether it's failed or not. assertEquals("Update transaction FAILED.", Status.OK, riakClient.update(bucket, keyPrefix + Integer.toString(updateKeyNumber), StringByteIterator .getByteIteratorMap(updateValue))); // Then, read the key again and... HashMap readValueAfterUpdate = new HashMap<>(); assertEquals("Update test FAILED. Unable to read key value.", Status.OK, riakClient.read(bucket, keyPrefix + Integer.toString(updateKeyNumber), null, readValueAfterUpdate)); // ...compare the result with the new one! assertEquals("Update transaction NOT EXECUTED PROPERLY. Values DID NOT CHANGE.", updateValue.toString(), readValueAfterUpdate.toString()); // Finally, restore the previously read key. assertEquals("Update test FAILED. Unable to restore previous key value.", Status.OK, riakClient.update(bucket, keyPrefix + Integer.toString(updateKeyNumber), readValueBeforeUpdate)); } /** * Test method for insert transaction. It is designed to insert a key just after the last key inserted in the setUp() * phase. */ @Test public void testInsert() { // Define a HashMap to insert and another one for the comparison operation. HashMap insertValue = createExpectedHashMap(recordsToInsert); HashMap readValue = new HashMap<>(); // Check whether the insertion transaction was performed or not. assertEquals("Insert transaction FAILED.", Status.OK, riakClient.insert(bucket, keyPrefix + Integer.toString(recordsToInsert), StringByteIterator. getByteIteratorMap(insertValue))); // Finally, compare the insertion performed with the one expected by reading the key. assertEquals("Insert test FAILED. Unable to read inserted value.", Status.OK, riakClient.read(bucket, keyPrefix + Integer.toString(recordsToInsert), null, readValue)); assertEquals("Insert test FAILED. Actual read transaction value is NOT MATCHING the inserted one.", insertValue.toString(), readValue.toString()); } /** * Test method for delete transaction. The test deletes a key, then performs a read that should give a * Status.NOT_FOUND response. Finally, it restores the previously read key. */ @Test public void testDelete() { // Choose a random key to delete, among the available ones. int deleteKeyNumber = new Random().nextInt(recordsToInsert); // Define a HashMap to save the previously stored values for its eventual restore. HashMap readValueBeforeDelete = new HashMap<>(); riakClient.read(bucket, keyPrefix + Integer.toString(deleteKeyNumber), null, readValueBeforeDelete); // First of all, delete the key. assertEquals("Delete transaction FAILED.", Status.OK, delete(keyPrefix + Integer.toString(deleteKeyNumber))); // Then, check if the deletion was actually achieved. assertEquals("Delete test FAILED. Key NOT deleted.", Status.NOT_FOUND, riakClient.read(bucket, keyPrefix + Integer.toString(deleteKeyNumber), null, null)); // Finally, restore the previously deleted key. assertEquals("Delete test FAILED. Unable to restore previous key value.", Status.OK, riakClient.insert(bucket, keyPrefix + Integer.toString(deleteKeyNumber), readValueBeforeDelete)); } private static Status delete(String key) { return riakClient.delete(bucket, key); } private static HashMap createExpectedHashMap(int value) { HashMap values = new HashMap<>(); values.put(firstField, Integer.toString(value)); values.put(secondField, Integer.toString(2 * value)); values.put(thirdField, Integer.toString(value * value)); return values; } } ================================================ FILE: rocksdb/README.md ================================================ ## Quick Start This section describes how to run YCSB on Redis. ### 1. Set the paths of dependencies in pom.xml Set the path of rocksdbjni-{version}-{platform}.jar. Set the path of librocksdbjni-{platform}.so. ### 2. Install Java and Maven ### 3. Set Up YCSB Git clone YCSB and compile: git clone http://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:rocksdb-binding -am clean package ### 5. Load data and run tests Load the data: ./bin/ycsb load rocksdb -s -P workloads/workloada > outputLoad.txt Run the workload test: ./bin/ycsb run rocksdb -s -P workloads/workloada > outputRun.txt Add rocksdb-binding for ycsb. ================================================ FILE: rocksdb/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.13.0-SNAPSHOT ../binding-parent rocksdb-binding RocksDB Binding jar org.rocksdb rocksdbjni system 5.7.2-linux64 /home/compaction/rocksdb-5.7.2-slot/java/target/rocksdbjni-5.7.2-linux64.jar org.rocksdb librocksdbjni system linux64 so /home/compaction/rocksdb-5.7.2-slot/java/target/librocksdbjni-linux64.so com.yahoo.ycsb core ${project.version} provided ================================================ FILE: rocksdb/src/main/java/RocksdbClient.java ================================================ import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import org.rocksdb.*; import java.nio.ByteBuffer; import java.util.*; /** * Created by Yosub on 10/2/14. * Modified by whb on 2017.10 */ public class RocksdbClient extends DB { static { RocksDB.loadLibrary(); } private static final String DB_PATH = "/tmp/rocksdb_slot_modified_statsdump_100s_ycsb_1G_60G_onlyrun"; private static final int BYTE_BUFFER_SIZE = 4096; private Date date; private RocksDB db; private Options options; public void init() throws DBException { System.out.println("Initializing RocksDB..."); date = new Date(); String dbPath = DB_PATH; options = new Options(); options.setCreateIfMissing(true); options.setStatsDumpPeriodSec(100); options.setCompressionType(CompressionType.NO_COMPRESSION); System.out.println("options.statsDumpPeriodSec() = " + options.statsDumpPeriodSec()); try { db = RocksDB.open(options, dbPath); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); assert(false); } System.out.println("Initializing RocksDB is over"); } public void cleanup() throws DBException { super.cleanup(); try { System.out.println("end operation"); String str = db.getProperty("rocksdb.stats"); System.out.println(str); // System.out.println("Begin full compaction"); // db.compactRange(); // str = db.getProperty("rocksdb.stats"); // System.out.println(str); } catch (RocksDBException e) { throw new DBException("Error while trying to print RocksDB statistics"); } System.out.println("Beginning Sleep..."); try{ for(int i=1; i<=10; i++){ System.out.println("begin the "+i+" interval :"); Thread.sleep(1000*60); String str = db.getProperty("rocksdb.stats"); System.out.println("after "+i+" mins"); System.out.println(str); } } catch (RocksDBException e) { throw new DBException("Error while trying to print RocksDB statistics while sleeping"); } catch (InterruptedException e){ e.printStackTrace(); System.out.println("Exception!!!! sleep found exception"); } System.out.println("Disconnecting RocksDB database..."); db.close(); } @Override public Status read(String table, String key, Set fields, Map result) { try { byte[] value = db.get(key.getBytes()); Map deserialized = deserialize(value); result.putAll(deserialized); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); return Status.ERROR; } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { System.out.println("Scan called! NOP for now"); return Status.OK; } @Override public Status update(String table, String key, Map values) { try { byte[] serialized = serialize(values); db.put(key.getBytes(), serialized); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); return Status.ERROR; } return Status.OK; } @Override public Status insert(String table, String key, Map values) { try { byte[] serialized = serialize(values); db.put(key.getBytes(), serialized); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); return Status.ERROR; } return Status.OK; } @Override public Status delete(String table, String key) { try { db.remove(key.getBytes()); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); return Status.ERROR; } return Status.OK; } private byte[] serialize(Map values) { ByteBuffer buf = ByteBuffer.allocate(BYTE_BUFFER_SIZE); // Number of elements in HashMap (int) buf.put((byte) values.size()); for (Map.Entry entry : values.entrySet()) { // Key string length (int) buf.put((byte) entry.getKey().length()); // Key bytes buf.put(entry.getKey().getBytes()); // Value bytes length (long) buf.put((byte) entry.getValue().bytesLeft()); // Value bytes buf.put((entry.getValue().toArray())); } byte[] result = new byte[buf.position()]; buf.get(result, 0, buf.position()); return result; } private HashMap deserialize(byte[] bytes) { HashMap result = new HashMap(); ByteBuffer buf = ByteBuffer.wrap(bytes); int count = buf.getInt(); for (int i = 0; i < count; i++) { int keyLength = buf.getInt(); byte[] keyBytes = new byte[keyLength]; buf.get(keyBytes, buf.position(), keyLength); int valueLength = buf.getInt(); byte[] valueBytes = new byte[valueLength]; buf.get(valueBytes, buf.position(), valueLength); result.put(new String(keyBytes), new ByteArrayByteIterator(valueBytes)); } return result; } } ================================================ FILE: s3/README.md ================================================ Quick Start =============== ### 1. Set Up YCSB Download the YCSB from this website: https://github.com/brianfrankcooper/YCSB/releases/ You can choose to download either the full stable version or just one of the available binding. ### 2. Configuration of the AWS credentials The access key ID and secret access key as well as the endPoint and region and the Client configurations like the maxErrorRetry can be set in a properties file under s3-binding/conf/s3.properties or sent by command line (see below). It is highly suggested to use the property file instead of to send the credentials through the command line. ### 3. Run YCSB To execute the benchmark using the S3 storage binding, first files must be uploaded using the "load" option with this command: ./bin/ycsb load s3 -p table=theBucket -p s3.endPoint=s3.amazonaws.com -p s3.accessKeyId=yourAccessKeyId -p s3.secretKey=yourSecretKey -p fieldlength=10 -p fieldcount=20 -P workloads/workloada With this command, the workload A will be executing with the loading phase. The file size is determined by the number of fields (fieldcount) and by the field size (fieldlength). In this case each file is 200 bytes (10 bytes for each field multiplied by 20 fields). Running the command: ./bin/ycsb -t s3 -p table=theBucket -p s3.endPoint=s3.amazonaws.com -p s3.accessKeyId=yourAccessKeyId -p s3.secretKey=yourSecretKey -p fieldlength=10 -p fieldcount=20 -P workloads/workloada the workload A will be executed with file size 200 bytes. #### S3 Storage Configuration Parameters The parameters to configure the S3 client can be set using the file "s3-binding/conf/s3.properties". This is highly advisable for the parameters s3.accessKeyId and s3.secretKey. All the other parameters can be set also on the command line. Here the list of all the parameters that is possible to configure: - `table` - This should be a S3 Storage bucket name and it replace the standard table name assigned by YCSB. - `s3.endpoint` - This indicate the endpoint used to connect to the S3 Storage service. - Default value is `s3.amazonaws.com`. - `s3.region` - This indicate the region where your buckets are. - Default value is `us-east-1`. - `s3.accessKeyId` - This is the accessKey of your S3 account. - `s3.secretKey` - This is the secret associated with your S3 account. - `s3.maxErrorRetry` - This is the maxErrorRetry parameter for the S3Client. - `s3.protocol` - This is the protocol parameter for the S3Client. The default value is HTTPS. - `s3.sse` - This parameter set to true activates the Server Side Encryption. - `s3.ssec` - This parameter if not null activates the SSE-C client side encryption. The value passed with this parameter is the client key used to encrpyt the files. ================================================ FILE: s3/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent s3-binding S3 Storage Binding jar com.amazonaws aws-java-sdk-s3 ${s3.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: s3/src/main/conf/s3.properties ================================================ # # Sample S3 configuration properties # # You may either set properties here or via the command line. # # the AWS S3 access key ID s3.accessKeyId=yourKey # the AWS S3 secret access key ID s3.secretKey=YourSecret # the AWS endpoint s3.endpoint=s3.amazonaws.com # activating the SSE server side encryption if true s3.sse=false # activating the SSE-C client side encryption if used #s3.ssec=U2CccCI40he2mZtg2aCEzofP7nQsfy4nP14VSYu6bFA= # set the protocol to use for the Client, default is HTTPS #s3.protocol=HTTPS # set the maxConnections to use for the Client, it should be not less than the # threads since only one client is created and shared between threads #s3.maxConnections= # set the maxErrorRetry parameter to use for the Client #s3.maxErrorRetry= ================================================ FILE: s3/src/main/java/com/yahoo/ycsb/db/S3Client.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. * * S3 storage client binding for YCSB. */ package com.yahoo.ycsb.db; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.net.*; import com.amazonaws.util.IOUtils; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.*; import com.amazonaws.auth.*; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.ClientConfiguration; import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; import com.amazonaws.Protocol; import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.SSECustomerKey; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; /** * S3 Storage client for YCSB framework. * * Properties to set: * * s3.accessKeyId=access key S3 aws * s3.secretKey=secret key S3 aws * s3.endPoint=s3.amazonaws.com * s3.region=us-east-1 * The parameter table is the name of the Bucket where to upload the files. * This must be created before to start the benchmark * The size of the file to upload is determined by two parameters: * - fieldcount this is the number of fields of a record in YCSB * - fieldlength this is the size in bytes of a single field in the record * together these two parameters define the size of the file to upload, * the size in bytes is given by the fieldlength multiplied by the fieldcount. * The name of the file is determined by the parameter key. *This key is automatically generated by YCSB. * */ public class S3Client extends DB { private static AmazonS3Client s3Client; private static String sse; private static SSECustomerKey ssecKey; private static final AtomicInteger INIT_COUNT = new AtomicInteger(0); /** * Cleanup any state for this storage. * Called once per S3 instance; */ @Override public void cleanup() throws DBException { if (INIT_COUNT.decrementAndGet() == 0) { try { s3Client.shutdown(); System.out.println("The client is shutdown successfully"); } catch (Exception e){ System.err.println("Could not shutdown the S3Client: "+e.toString()); e.printStackTrace(); } finally { if (s3Client != null){ s3Client = null; } } } } /** * Delete a file from S3 Storage. * * @param bucket * The name of the bucket * @param key * The record key of the file to delete. * @return OK on success, otherwise ERROR. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status delete(String bucket, String key) { try { s3Client.deleteObject(new DeleteObjectRequest(bucket, key)); } catch (Exception e){ System.err.println("Not possible to delete the key "+key); e.printStackTrace(); return Status.ERROR; } return Status.OK; } /** * Initialize any state for the storage. * Called once per S3 instance; If the client is not null it is re-used. */ @Override public void init() throws DBException { final int count = INIT_COUNT.incrementAndGet(); synchronized (S3Client.class){ Properties propsCL = getProperties(); int recordcount = Integer.parseInt( propsCL.getProperty("recordcount")); int operationcount = Integer.parseInt( propsCL.getProperty("operationcount")); int numberOfOperations = 0; if (recordcount > 0){ if (recordcount > operationcount){ numberOfOperations = recordcount; } else { numberOfOperations = operationcount; } } else { numberOfOperations = operationcount; } if (count <= numberOfOperations) { String accessKeyId = null; String secretKey = null; String endPoint = null; String region = null; String maxErrorRetry = null; String maxConnections = null; String protocol = null; BasicAWSCredentials s3Credentials; ClientConfiguration clientConfig; if (s3Client != null) { System.out.println("Reusing the same client"); return; } try { InputStream propFile = S3Client.class.getClassLoader() .getResourceAsStream("s3.properties"); Properties props = new Properties(System.getProperties()); props.load(propFile); accessKeyId = props.getProperty("s3.accessKeyId"); if (accessKeyId == null){ accessKeyId = propsCL.getProperty("s3.accessKeyId"); } System.out.println(accessKeyId); secretKey = props.getProperty("s3.secretKey"); if (secretKey == null){ secretKey = propsCL.getProperty("s3.secretKey"); } System.out.println(secretKey); endPoint = props.getProperty("s3.endPoint"); if (endPoint == null){ endPoint = propsCL.getProperty("s3.endPoint", "s3.amazonaws.com"); } System.out.println(endPoint); region = props.getProperty("s3.region"); if (region == null){ region = propsCL.getProperty("s3.region", "us-east-1"); } System.out.println(region); maxErrorRetry = props.getProperty("s3.maxErrorRetry"); if (maxErrorRetry == null){ maxErrorRetry = propsCL.getProperty("s3.maxErrorRetry", "15"); } maxConnections = props.getProperty("s3.maxConnections"); if (maxConnections == null){ maxConnections = propsCL.getProperty("s3.maxConnections"); } protocol = props.getProperty("s3.protocol"); if (protocol == null){ protocol = propsCL.getProperty("s3.protocol", "HTTPS"); } sse = props.getProperty("s3.sse"); if (sse == null){ sse = propsCL.getProperty("s3.sse", "false"); } String ssec = props.getProperty("s3.ssec"); if (ssec == null){ ssec = propsCL.getProperty("s3.ssec", null); } else { ssecKey = new SSECustomerKey(ssec); } } catch (Exception e){ System.err.println("The file properties doesn't exist "+e.toString()); e.printStackTrace(); } try { System.out.println("Inizializing the S3 connection"); s3Credentials = new BasicAWSCredentials(accessKeyId, secretKey); clientConfig = new ClientConfiguration(); clientConfig.setMaxErrorRetry(Integer.parseInt(maxErrorRetry)); if(protocol.equals("HTTP")) { clientConfig.setProtocol(Protocol.HTTP); } else { clientConfig.setProtocol(Protocol.HTTPS); } if(maxConnections != null) { clientConfig.setMaxConnections(Integer.parseInt(maxConnections)); } s3Client = new AmazonS3Client(s3Credentials, clientConfig); s3Client.setRegion(Region.getRegion(Regions.fromName(region))); s3Client.setEndpoint(endPoint); System.out.println("Connection successfully initialized"); } catch (Exception e){ System.err.println("Could not connect to S3 storage: "+ e.toString()); e.printStackTrace(); throw new DBException(e); } } else { System.err.println( "The number of threads must be less or equal than the operations"); throw new DBException(new Error( "The number of threads must be less or equal than the operations")); } } } /** * Create a new File in the Bucket. Any field/value pairs in the specified * values HashMap will be written into the file with the specified record * key. * * @param bucket * The name of the bucket * @param key * The record key of the file to insert. * @param values * A HashMap of field/value pairs to insert in the file. * Only the content of the first field is written to a byteArray * multiplied by the number of field. In this way the size * of the file to upload is determined by the fieldlength * and fieldcount parameters. * @return OK on success, ERROR otherwise. See the * {@link DB} class's description for a discussion of error codes. */ @Override public Status insert(String bucket, String key, Map values) { return writeToStorage(bucket, key, values, true, sse, ssecKey); } /** * Read a file from the Bucket. Each field/value pair from the result * will be stored in a HashMap. * * @param bucket * The name of the bucket * @param key * The record key of the file to read. * @param fields * The list of fields to read, or null for all of them, * it is null by default * @param result * A HashMap of field/value pairs for the result * @return OK on success, ERROR otherwise. */ @Override public Status read(String bucket, String key, Set fields, Map result) { return readFromStorage(bucket, key, result, ssecKey); } /** * Update a file in the database. Any field/value pairs in the specified * values HashMap will be written into the file with the specified file * key, overwriting any existing values with the same field name. * * @param bucket * The name of the bucket * @param key * The file key of the file to write. * @param values * A HashMap of field/value pairs to update in the record * @return OK on success, ERORR otherwise. */ @Override public Status update(String bucket, String key, Map values) { return writeToStorage(bucket, key, values, false, sse, ssecKey); } /** * Perform a range scan for a set of files in the bucket. Each * field/value pair from the result will be stored in a HashMap. * * @param bucket * The name of the bucket * @param startkey * The file key of the first file to read. * @param recordcount * The number of files to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one file * @return OK on success, ERROR otherwise. */ @Override public Status scan(String bucket, String startkey, int recordcount, Set fields, Vector> result) { return scanFromStorage(bucket, startkey, recordcount, result, ssecKey); } /** * Upload a new object to S3 or update an object on S3. * * @param bucket * The name of the bucket * @param key * The file key of the object to upload/update. * @param values * The data to be written on the object * @param updateMarker * A boolean value. If true a new object will be uploaded * to S3. If false an existing object will be re-uploaded * */ protected Status writeToStorage(String bucket, String key, Map values, Boolean updateMarker, String sseLocal, SSECustomerKey ssecLocal) { int totalSize = 0; int fieldCount = values.size(); //number of fields to concatenate // getting the first field in the values Object keyToSearch = values.keySet().toArray()[0]; // getting the content of just one field byte[] sourceArray = values.get(keyToSearch).toArray(); int sizeArray = sourceArray.length; //size of each array if (updateMarker){ totalSize = sizeArray*fieldCount; } else { try { Map.Entry objectAndMetadata = getS3ObjectAndMetadata(bucket, key, ssecLocal); int sizeOfFile = (int)objectAndMetadata.getValue().getContentLength(); fieldCount = sizeOfFile/sizeArray; totalSize = sizeOfFile; objectAndMetadata.getKey().close(); } catch (Exception e){ System.err.println("Not possible to get the object :"+key); e.printStackTrace(); return Status.ERROR; } } byte[] destinationArray = new byte[totalSize]; int offset = 0; for (int i = 0; i < fieldCount; i++) { System.arraycopy(sourceArray, 0, destinationArray, offset, sizeArray); offset += sizeArray; } try (InputStream input = new ByteArrayInputStream(destinationArray)) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(totalSize); PutObjectRequest putObjectRequest = null; if (sseLocal.equals("true")) { metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); putObjectRequest = new PutObjectRequest(bucket, key, input, metadata); } else if (ssecLocal != null) { putObjectRequest = new PutObjectRequest(bucket, key, input, metadata).withSSECustomerKey(ssecLocal); } else { putObjectRequest = new PutObjectRequest(bucket, key, input, metadata); } try { PutObjectResult res = s3Client.putObject(putObjectRequest); if(res.getETag() == null) { return Status.ERROR; } else { if (sseLocal.equals("true")) { System.out.println("Uploaded object encryption status is " + res.getSSEAlgorithm()); } else if (ssecLocal != null) { System.out.println("Uploaded object encryption status is " + res.getSSEAlgorithm()); } } } catch (Exception e) { System.err.println("Not possible to write object :"+key); e.printStackTrace(); return Status.ERROR; } } catch (Exception e) { System.err.println("Error in the creation of the stream :"+e.toString()); e.printStackTrace(); return Status.ERROR; } return Status.OK; } /** * Download an object from S3. * * @param bucket * The name of the bucket * @param key * The file key of the object to upload/update. * @param result * The Hash map where data from the object are written * */ protected Status readFromStorage(String bucket, String key, Map result, SSECustomerKey ssecLocal) { try { Map.Entry objectAndMetadata = getS3ObjectAndMetadata(bucket, key, ssecLocal); InputStream objectData = objectAndMetadata.getKey().getObjectContent(); //consuming the stream // writing the stream to bytes and to results result.put(key, new ByteArrayByteIterator(IOUtils.toByteArray(objectData))); objectData.close(); objectAndMetadata.getKey().close(); } catch (Exception e){ System.err.println("Not possible to get the object "+key); e.printStackTrace(); return Status.ERROR; } return Status.OK; } private Map.Entry getS3ObjectAndMetadata(String bucket, String key, SSECustomerKey ssecLocal) { GetObjectRequest getObjectRequest; GetObjectMetadataRequest getObjectMetadataRequest; if (ssecLocal != null) { getObjectRequest = new GetObjectRequest(bucket, key).withSSECustomerKey(ssecLocal); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key).withSSECustomerKey(ssecLocal); } else { getObjectRequest = new GetObjectRequest(bucket, key); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key); } return new AbstractMap.SimpleEntry<>(s3Client.getObject(getObjectRequest), s3Client.getObjectMetadata(getObjectMetadataRequest)); } /** * Perform an emulation of a database scan operation on a S3 bucket. * * @param bucket * The name of the bucket * @param startkey * The file key of the first file to read. * @param recordcount * The number of files to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one file * */ protected Status scanFromStorage(String bucket, String startkey, int recordcount, Vector> result, SSECustomerKey ssecLocal) { int counter = 0; ObjectListing listing = s3Client.listObjects(bucket); List summaries = listing.getObjectSummaries(); List keyList = new ArrayList(); int startkeyNumber = 0; int numberOfIteration = 0; // getting the list of files in the bucket while (listing.isTruncated()) { listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries()); } for (S3ObjectSummary summary : summaries) { String summaryKey = summary.getKey(); keyList.add(summaryKey); } // Sorting the list of files in Alphabetical order Collections.sort(keyList); // sorting the list // Getting the position of the startingfile for the scan for (String key : keyList) { if (key.equals(startkey)){ startkeyNumber = counter; } else { counter = counter + 1; } } // Checking if the total number of file is bigger than the file to read, // if not using the total number of Files if (recordcount < keyList.size()) { numberOfIteration = recordcount; } else { numberOfIteration = keyList.size(); } // Reading the Files starting from the startkey File till the end // of the Files or Till the recordcount number for (int i = startkeyNumber; i < numberOfIteration; i++){ HashMap resultTemp = new HashMap(); readFromStorage(bucket, keyList.get(i), resultTemp, ssecLocal); result.add(resultTemp); } return Status.OK; } } ================================================ FILE: s3/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /** * Copyright (c) 2015 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. * * S3 storage client binding for YCSB. */ package com.yahoo.ycsb.db; ================================================ FILE: solr/README.md ================================================ ## Quick Start This section describes how to run YCSB on Solr running locally. ### 1. Set Up YCSB Clone the YCSB git repository and compile: git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:solr-binding -am clean package ### 2. Set Up Solr There must be a running Solr instance with a core/collection pre-defined and configured. - See this [API](https://cwiki.apache.org/confluence/display/solr/CoreAdmin+API#CoreAdminAPI-CREATE) reference on how to create a core. - See this [API](https://cwiki.apache.org/confluence/display/solr/Collections+API#CollectionsAPI-api1) reference on how to create a collection in SolrCloud mode. The `conf/schema.xml` configuration file present in the core/collection just created must be configured to handle the expected field names during benchmarking. Below illustrates a sample from a schema config file that matches the default field names used by the ycsb client: If running in SolrCloud mode ensure there is an external Zookeeper cluster running. - See [here](https://cwiki.apache.org/confluence/display/solr/Setting+Up+an+External+ZooKeeper+Ensemble) for details on how to set up an external Zookeeper cluster. - See [here](https://cwiki.apache.org/confluence/display/solr/Using+ZooKeeper+to+Manage+Configuration+Files) for instructions on how to use Zookeeper to manage your core/collection configuration files. ### 3. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load solr -s -P workloads/workloada -p table= Then, run the workload: ./bin/ycsb run solr -s -P workloads/workloada -p table= For further configuration see below: ### Default Configuration Parameters The default settings for the Solr node that is created is as follows: - `solr.cloud` - A Boolean value indicating if Solr is running in SolrCloud mode. If so there must be an external Zookeeper cluster running also. - Default value is `false` and therefore expects solr to be running in stand-alone mode. - `solr.base.url` - The base URL in which to interface with a running Solr instance in stand-alone mode - Default value is `http://localhost:8983/solr - `solr.commit.within.time` - The max time in ms to wait for a commit when in batch mode, ignored otherwise - Default value is `1000ms` - `solr.batch.mode` - Indicates if inserts/updates/deletes should be commited in batches (frequency controlled by the `solr.commit.within.time` parameter) or commit 1 document at a time. - Default value is `false` - `solr.zookeeper.hosts` - A list of comma seperated host:port pairs of Zookeeper nodes used to manage SolrCloud configurations. - Must be passed when in [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) mode. - Default value is `localhost:2181` ### Custom Configuration If you wish to customize the settings used to create the Solr node you can created a new property file that contains your desired Solr node settings and pass it in via the parameter to 'bin/ycsb' script. Note that the default properties will be kept if you don't explicitly overwrite them. Assuming that we have a properties file named "myproperties.data" that contains custom Solr node configuration you can execute the following to pass it into the Solr client: ./bin/ycsb run solr -P workloads/workloada -P myproperties.data -s If you wish to use SolrCloud mode ensure a Solr cluster is running with an external zookeeper cluster and an appropriate collection has been created. Make sure to pass the following properties as parameters to 'bin/ycsb' script. solr.cloud=true solr.zookeeper.hosts=:,...,: ================================================ FILE: solr/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent solr-binding Solr Binding jar true com.yahoo.ycsb core ${project.version} provided org.apache.solr solr-solrj ${solr.version} commons-codec commons-codec 1.10 org.slf4j slf4j-log4j12 1.7.21 org.apache.solr solr-test-framework ${solr.version} test jdk.tools jdk.tools ================================================ FILE: solr/src/main/java/com/yahoo/ycsb/db/solr/SolrClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; import java.io.IOException; import java.util.*; import java.util.Map.Entry; /** * Solr client for YCSB framework. * *

* Default properties to set: *

*
    * See README.md *
* */ public class SolrClient extends DB { public static final String DEFAULT_CLOUD_MODE = "false"; public static final String DEFAULT_BATCH_MODE = "false"; public static final String DEFAULT_ZOOKEEPER_HOSTS = "localhost:2181"; public static final String DEFAULT_SOLR_BASE_URL = "http://localhost:8983/solr"; public static final String DEFAULT_COMMIT_WITHIN_TIME = "1000"; private org.apache.solr.client.solrj.SolrClient client; private Integer commitTime; private Boolean batchMode; /** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per * client thread. */ @Override public void init() throws DBException { Properties props = getProperties(); commitTime = Integer .parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME)); batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE)); String jaasConfPath = props.getProperty("solr.jaas.conf.path"); if(jaasConfPath != null) { System.setProperty("java.security.auth.login.config", jaasConfPath); HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); } // Check if Solr cluster is running in SolrCloud or Stand-alone mode Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE)); System.err.println("Solr Cloud Mode = " + cloudMode); if (cloudMode) { System.err.println("Solr Zookeeper Remote Hosts = " + props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); client = new CloudSolrClient( props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); } else { client = new HttpSolrClient(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)); } } @Override public void cleanup() throws DBException { try { client.close(); } catch (IOException e) { throw new DBException(e); } } /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be * written into the record with the specified record key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status insert(String table, String key, Map values) { try { SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", key); for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { doc.addField(entry.getKey(), entry.getValue()); } UpdateResponse response; if (batchMode) { response = client.add(table, doc, commitTime); } else { response = client.add(table, doc); client.commit(table); } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status delete(String table, String key) { try { UpdateResponse response; if (batchMode) { response = client.deleteById(table, key, commitTime); } else { response = client.deleteById(table, key); client.commit(table); } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Read a record from the database. Each field/value pair from the result will be stored in a * HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public Status read(String table, String key, Set fields, Map result) { try { Boolean returnFields = false; String[] fieldList = null; if (fields != null) { returnFields = true; fieldList = fields.toArray(new String[fields.size()]); } SolrQuery query = new SolrQuery(); query.setQuery("id:" + key); if (returnFields) { query.setFields(fieldList); } final QueryResponse response = client.query(table, query); SolrDocumentList results = response.getResults(); if ((results != null) && (results.getNumFound() > 0)) { for (String field : results.get(0).getFieldNames()) { result.put(field, new StringByteIterator(String.valueOf(results.get(0).getFirstValue(field)))); } } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be * written into the record with the specified record key, overwriting any existing values with the * same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status update(String table, String key, Map values) { try { SolrInputDocument updatedDoc = new SolrInputDocument(); updatedDoc.addField("id", key); for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { updatedDoc.addField(entry.getKey(), Collections.singletonMap("set", entry.getValue())); } UpdateResponse writeResponse; if (batchMode) { writeResponse = client.add(table, updatedDoc, commitTime); } else { writeResponse = client.add(table, updatedDoc); client.commit(table); } return checkStatus(writeResponse.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Perform a range scan for a set of records in the database. Each field/value pair from the * result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { Boolean returnFields = false; String[] fieldList = null; if (fields != null) { returnFields = true; fieldList = fields.toArray(new String[fields.size()]); } SolrQuery query = new SolrQuery(); query.setQuery("*:*"); query.setParam("fq", "id:[ " + startkey + " TO * ]"); if (returnFields) { query.setFields(fieldList); } query.setRows(recordcount); final QueryResponse response = client.query(table, query); SolrDocumentList results = response.getResults(); HashMap entry; for (SolrDocument hit : results) { entry = new HashMap((int) results.getNumFound()); for (String field : hit.getFieldNames()) { entry.put(field, new StringByteIterator(String.valueOf(hit.getFirstValue(field)))); } result.add(entry); } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } private Status checkStatus(int status) { Status responseStatus; switch (status) { case 0: responseStatus = Status.OK; break; case 400: responseStatus = Status.BAD_REQUEST; break; case 403: responseStatus = Status.FORBIDDEN; break; case 404: responseStatus = Status.NOT_FOUND; break; case 500: responseStatus = Status.ERROR; break; case 503: responseStatus = Status.SERVICE_UNAVAILABLE; break; default: responseStatus = Status.UNEXPECTED_STATE; break; } return responseStatus; } } ================================================ FILE: solr/src/main/java/com/yahoo/ycsb/db/solr/package-info.java ================================================ /* * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for * Solr. */ package com.yahoo.ycsb.db.solr; ================================================ FILE: solr/src/main/resources/log4j.properties ================================================ # Root logger option log4j.rootLogger=INFO, stderr # Direct log messages to stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.Target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n ================================================ FILE: solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientBaseTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.util.NamedList; import org.junit.*; import java.io.File; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public abstract class SolrClientBaseTest { protected static MiniSolrCloudCluster miniSolrCloudCluster; private DB instance; private final static HashMap MOCK_DATA; protected final static String MOCK_TABLE = "ycsb"; private final static String MOCK_KEY0 = "0"; private final static String MOCK_KEY1 = "1"; private final static int NUM_RECORDS = 10; static { MOCK_DATA = new HashMap<>(NUM_RECORDS); for (int i = 0; i < NUM_RECORDS; i++) { MOCK_DATA.put("field" + i, new StringByteIterator("value" + i)); } } @BeforeClass public static void onlyOnce() throws Exception { Path miniSolrCloudClusterTempDirectory = Files.createTempDirectory("miniSolrCloudCluster"); miniSolrCloudClusterTempDirectory.toFile().deleteOnExit(); miniSolrCloudCluster = new MiniSolrCloudCluster(1, miniSolrCloudClusterTempDirectory, JettyConfig.builder().build()); // Upload Solr configuration URL configDir = SolrClientBaseTest.class.getClassLoader().getResource("solr_config"); assertNotNull(configDir); miniSolrCloudCluster.uploadConfigDir(new File(configDir.toURI()), MOCK_TABLE); } @AfterClass public static void destroy() throws Exception { if(miniSolrCloudCluster != null) { miniSolrCloudCluster.shutdown(); } } @Before public void setup() throws Exception { NamedList namedList = miniSolrCloudCluster.createCollection(MOCK_TABLE, 1, 1, MOCK_TABLE, null); assertEquals(namedList.indexOf("success", 0), 1); Thread.sleep(1000); instance = getDB(); } @After public void tearDown() throws Exception { if(miniSolrCloudCluster != null) { NamedList namedList = miniSolrCloudCluster.deleteCollection(MOCK_TABLE); assertEquals(namedList.indexOf("success", 0), 1); Thread.sleep(1000); } } @Test public void testInsert() throws Exception { Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA); assertEquals(Status.OK, result); } @Test public void testDelete() throws Exception { Status result = instance.delete(MOCK_TABLE, MOCK_KEY1); assertEquals(Status.OK, result); } @Test public void testRead() throws Exception { Set fields = MOCK_DATA.keySet(); HashMap resultParam = new HashMap<>(NUM_RECORDS); Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam); assertEquals(Status.OK, result); } @Test public void testUpdate() throws Exception { HashMap newValues = new HashMap<>(NUM_RECORDS); for (int i = 0; i < NUM_RECORDS; i++) { newValues.put("field" + i, new StringByteIterator("newvalue" + i)); } Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues); assertEquals(Status.OK, result); //validate that the values changed HashMap resultParam = new HashMap<>(NUM_RECORDS); instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam); for (int i = 0; i < NUM_RECORDS; i++) { assertEquals("newvalue" + i, resultParam.get("field" + i).toString()); } } @Test public void testScan() throws Exception { Set fields = MOCK_DATA.keySet(); Vector> resultParam = new Vector<>(NUM_RECORDS); Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, NUM_RECORDS, fields, resultParam); assertEquals(Status.OK, result); } /** * Gets the test DB. * * @return The test DB. */ protected DB getDB() { return getDB(new Properties()); } /** * Gets the test DB. * * @param props * Properties to pass to the client. * @return The test DB. */ protected abstract DB getDB(Properties props); } ================================================ FILE: solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientCloudTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.DB; import org.junit.After; import java.util.Properties; import static org.junit.Assume.assumeNoException; public class SolrClientCloudTest extends SolrClientBaseTest { private SolrClient instance; @After public void tearDown() throws Exception { try { if(instance != null) { instance.cleanup(); } } finally { super.tearDown(); } } @Override protected DB getDB(Properties props) { instance = new SolrClient(); props.setProperty("solr.cloud", "true"); props.setProperty("solr.zookeeper.hosts", miniSolrCloudCluster.getSolrClient().getZkHost()); instance.setProperties(props); try { instance.init(); } catch (Exception error) { assumeNoException(error); } return instance; } } ================================================ FILE: solr/src/test/java/com/yahoo/ycsb/db/solr/SolrClientTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr; import com.yahoo.ycsb.DB; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.junit.After; import java.util.Properties; import static org.junit.Assume.assumeNoException; public class SolrClientTest extends SolrClientBaseTest { private SolrClient instance; @After public void tearDown() throws Exception { try { if(instance != null) { instance.cleanup(); } } finally { super.tearDown(); } } @Override protected DB getDB(Properties props) { instance = new SolrClient(); // Use the first Solr server in the cluster. // Doesn't matter if there are more since requests will be forwarded properly by Solr. JettySolrRunner jettySolrRunner = miniSolrCloudCluster.getJettySolrRunners().get(0); String solrBaseUrl = String.format("http://localhost:%s%s", jettySolrRunner.getLocalPort(), jettySolrRunner.getBaseUrl()); props.setProperty("solr.base.url", solrBaseUrl); instance.setProperties(props); try { instance.init(); } catch (Exception error) { assumeNoException(error); } return instance; } } ================================================ FILE: solr/src/test/resources/log4j.properties ================================================ # Copyright (c) 2015-2016 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=INFO, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZooKeeper log4j.logger.org.apache.zookeeper=ERROR # Solr classes are too chatty in test at INFO log4j.logger.org.apache.solr=ERROR log4j.logger.org.eclipse.jetty=ERROR ================================================ FILE: solr/src/test/resources/solr_config/schema.xml ================================================ text id ================================================ FILE: solr/src/test/resources/solr_config/solrconfig.xml ================================================ ${tests.luceneMatchVersion:LATEST} ${useCompoundFile:false} ${solr.data.dir:} ${solr.data.dir:} *:* all server-enabled.txt solr ================================================ FILE: solr6/README.md ================================================ ## Quick Start This section describes how to run YCSB on Solr running locally. ### 1. Set Up YCSB Clone the YCSB git repository and compile: git clone git://github.com/brianfrankcooper/YCSB.git cd YCSB mvn -pl com.yahoo.ycsb:solr6-binding -am clean package ### 2. Set Up Solr There must be a running Solr instance with a core/collection pre-defined and configured. - See this [API](https://cwiki.apache.org/confluence/display/solr/CoreAdmin+API#CoreAdminAPI-CREATE) reference on how to create a core. - See this [API](https://cwiki.apache.org/confluence/display/solr/Collections+API#CollectionsAPI-api1) reference on how to create a collection in SolrCloud mode. The `conf/schema.xml` configuration file present in the core/collection just created must be configured to handle the expected field names during benchmarking. Below illustrates a sample from a schema config file that matches the default field names used by the ycsb client: If running in SolrCloud mode ensure there is an external Zookeeper cluster running. - See [here](https://cwiki.apache.org/confluence/display/solr/Setting+Up+an+External+ZooKeeper+Ensemble) for details on how to set up an external Zookeeper cluster. - See [here](https://cwiki.apache.org/confluence/display/solr/Using+ZooKeeper+to+Manage+Configuration+Files) for instructions on how to use Zookeeper to manage your core/collection configuration files. ### 3. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load solr6 -s -P workloads/workloada -p table= Then, run the workload: ./bin/ycsb run solr6 -s -P workloads/workloada -p table= For further configuration see below: ### Default Configuration Parameters The default settings for the Solr node that is created is as follows: - `solr.cloud` - A Boolean value indicating if Solr is running in SolrCloud mode. If so there must be an external Zookeeper cluster running also. - Default value is `false` and therefore expects solr to be running in stand-alone mode. - `solr.base.url` - The base URL in which to interface with a running Solr instance in stand-alone mode - Default value is `http://localhost:8983/solr - `solr.commit.within.time` - The max time in ms to wait for a commit when in batch mode, ignored otherwise - Default value is `1000ms` - `solr.batch.mode` - Indicates if inserts/updates/deletes should be commited in batches (frequency controlled by the `solr.commit.within.time` parameter) or commit 1 document at a time. - Default value is `false` - `solr.zookeeper.hosts` - A list of comma seperated host:port pairs of Zookeeper nodes used to manage SolrCloud configurations. - Must be passed when in [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) mode. - Default value is `localhost:2181` ### Custom Configuration If you wish to customize the settings used to create the Solr node you can created a new property file that contains your desired Solr node settings and pass it in via the parameter to 'bin/ycsb' script. Note that the default properties will be kept if you don't explicitly overwrite them. Assuming that we have a properties file named "myproperties.data" that contains custom Solr node configuration you can execute the following to pass it into the Solr client: ./bin/ycsb run solr6 -P workloads/workloada -P myproperties.data -s If you wish to use SolrCloud mode ensure a Solr cluster is running with an external zookeeper cluster and an appropriate collection has been created. Make sure to pass the following properties as parameters to 'bin/ycsb' script. solr.cloud=true solr.zookeeper.hosts=:,...,: ================================================ FILE: solr6/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent solr6-binding Solr 6 Binding jar true com.yahoo.ycsb core ${project.version} provided org.apache.solr solr-solrj ${solr6.version} commons-codec commons-codec 1.10 org.slf4j slf4j-log4j12 1.7.21 org.apache.solr solr-test-framework ${solr6.version} test jdk.tools jdk.tools jdk8-tests 1.8 false ================================================ FILE: solr6/src/main/java/com/yahoo/ycsb/db/solr6/SolrClient.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr6; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; import java.io.IOException; import java.util.*; import java.util.Map.Entry; /** * Solr client for YCSB framework. * *

* Default properties to set: *

*
    * See README.md *
* */ public class SolrClient extends DB { public static final String DEFAULT_CLOUD_MODE = "false"; public static final String DEFAULT_BATCH_MODE = "false"; public static final String DEFAULT_ZOOKEEPER_HOSTS = "localhost:2181"; public static final String DEFAULT_SOLR_BASE_URL = "http://localhost:8983/solr"; public static final String DEFAULT_COMMIT_WITHIN_TIME = "1000"; private org.apache.solr.client.solrj.SolrClient client; private Integer commitTime; private Boolean batchMode; /** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per * client thread. */ @Override public void init() throws DBException { Properties props = getProperties(); commitTime = Integer .parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME)); batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE)); String jaasConfPath = props.getProperty("solr.jaas.conf.path"); if(jaasConfPath != null) { System.setProperty("java.security.auth.login.config", jaasConfPath); HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); } // Check if Solr cluster is running in SolrCloud or Stand-alone mode Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE)); System.err.println("Solr Cloud Mode = " + cloudMode); if (cloudMode) { System.err.println("Solr Zookeeper Remote Hosts = " + props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); client = new CloudSolrClient.Builder().withZkHost( Arrays.asList(props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS).split(","))).build(); } else { client = new HttpSolrClient.Builder(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)).build(); } } @Override public void cleanup() throws DBException { try { client.close(); } catch (IOException e) { throw new DBException(e); } } /** * Insert a record in the database. Any field/value pairs in the specified values HashMap will be * written into the record with the specified record key. * * @param table * The name of the table * @param key * The record key of the record to insert. * @param values * A HashMap of field/value pairs to insert in the record * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status insert(String table, String key, Map values) { try { SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", key); for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { doc.addField(entry.getKey(), entry.getValue()); } UpdateResponse response; if (batchMode) { response = client.add(table, doc, commitTime); } else { response = client.add(table, doc); client.commit(table); } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Delete a record from the database. * * @param table * The name of the table * @param key * The record key of the record to delete. * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status delete(String table, String key) { try { UpdateResponse response; if (batchMode) { response = client.deleteById(table, key, commitTime); } else { response = client.deleteById(table, key); client.commit(table); } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Read a record from the database. Each field/value pair from the result will be stored in a * HashMap. * * @param table * The name of the table * @param key * The record key of the record to read. * @param fields * The list of fields to read, or null for all of them * @param result * A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error or "not found". */ @Override public Status read(String table, String key, Set fields, Map result) { try { Boolean returnFields = false; String[] fieldList = null; if (fields != null) { returnFields = true; fieldList = fields.toArray(new String[fields.size()]); } SolrQuery query = new SolrQuery(); query.setQuery("id:" + key); if (returnFields) { query.setFields(fieldList); } final QueryResponse response = client.query(table, query); SolrDocumentList results = response.getResults(); if ((results != null) && (results.getNumFound() > 0)) { for (String field : results.get(0).getFieldNames()) { result.put(field, new StringByteIterator(String.valueOf(results.get(0).getFirstValue(field)))); } } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Update a record in the database. Any field/value pairs in the specified values HashMap will be * written into the record with the specified record key, overwriting any existing values with the * same field name. * * @param table * The name of the table * @param key * The record key of the record to write. * @param values * A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status update(String table, String key, Map values) { try { SolrInputDocument updatedDoc = new SolrInputDocument(); updatedDoc.addField("id", key); for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { updatedDoc.addField(entry.getKey(), Collections.singletonMap("set", entry.getValue())); } UpdateResponse writeResponse; if (batchMode) { writeResponse = client.add(table, updatedDoc, commitTime); } else { writeResponse = client.add(table, updatedDoc); client.commit(table); } return checkStatus(writeResponse.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } /** * Perform a range scan for a set of records in the database. Each field/value pair from the * result will be stored in a HashMap. * * @param table * The name of the table * @param startkey * The record key of the first record to read. * @param recordcount * The number of records to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return Zero on success, a non-zero error code on error. See this class's description for a * discussion of error codes. */ @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { try { Boolean returnFields = false; String[] fieldList = null; if (fields != null) { returnFields = true; fieldList = fields.toArray(new String[fields.size()]); } SolrQuery query = new SolrQuery(); query.setQuery("*:*"); query.setParam("fq", "id:[ " + startkey + " TO * ]"); if (returnFields) { query.setFields(fieldList); } query.setRows(recordcount); final QueryResponse response = client.query(table, query); SolrDocumentList results = response.getResults(); HashMap entry; for (SolrDocument hit : results) { entry = new HashMap<>((int) results.getNumFound()); for (String field : hit.getFieldNames()) { entry.put(field, new StringByteIterator(String.valueOf(hit.getFirstValue(field)))); } result.add(entry); } return checkStatus(response.getStatus()); } catch (IOException | SolrServerException e) { e.printStackTrace(); } return Status.ERROR; } private Status checkStatus(int status) { Status responseStatus; switch (status) { case 0: responseStatus = Status.OK; break; case 400: responseStatus = Status.BAD_REQUEST; break; case 403: responseStatus = Status.FORBIDDEN; break; case 404: responseStatus = Status.NOT_FOUND; break; case 500: responseStatus = Status.ERROR; break; case 503: responseStatus = Status.SERVICE_UNAVAILABLE; break; default: responseStatus = Status.UNEXPECTED_STATE; break; } return responseStatus; } } ================================================ FILE: solr6/src/main/java/com/yahoo/ycsb/db/solr6/package-info.java ================================================ /* * Copyright (c) 2016 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * The YCSB binding for * Solr. */ package com.yahoo.ycsb.db.solr6; ================================================ FILE: solr6/src/main/resources/log4j.properties ================================================ # Root logger option log4j.rootLogger=INFO, stderr # Direct log messages to stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.Target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n ================================================ FILE: solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientBaseTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr6; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.StringByteIterator; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.util.NamedList; import org.junit.*; import java.io.File; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Properties; import java.util.Set; import java.util.Vector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public abstract class SolrClientBaseTest { protected static MiniSolrCloudCluster miniSolrCloudCluster; private DB instance; private final static HashMap MOCK_DATA; protected final static String MOCK_TABLE = "ycsb"; private final static String MOCK_KEY0 = "0"; private final static String MOCK_KEY1 = "1"; private final static int NUM_RECORDS = 10; static { MOCK_DATA = new HashMap<>(NUM_RECORDS); for (int i = 0; i < NUM_RECORDS; i++) { MOCK_DATA.put("field" + i, new StringByteIterator("value" + i)); } } @BeforeClass public static void onlyOnce() throws Exception { Path miniSolrCloudClusterTempDirectory = Files.createTempDirectory("miniSolrCloudCluster"); miniSolrCloudClusterTempDirectory.toFile().deleteOnExit(); miniSolrCloudCluster = new MiniSolrCloudCluster(1, miniSolrCloudClusterTempDirectory, JettyConfig.builder().build()); // Upload Solr configuration URL configDir = SolrClientBaseTest.class.getClassLoader().getResource("solr_config"); assertNotNull(configDir); miniSolrCloudCluster.uploadConfigDir(new File(configDir.toURI()), MOCK_TABLE); } @AfterClass public static void destroy() throws Exception { if(miniSolrCloudCluster != null) { miniSolrCloudCluster.shutdown(); } } @Before public void setup() throws Exception { NamedList namedList = miniSolrCloudCluster.createCollection(MOCK_TABLE, 1, 1, MOCK_TABLE, null); assertEquals(namedList.indexOf("success", 0), 1); Thread.sleep(1000); instance = getDB(); } @After public void tearDown() throws Exception { if(miniSolrCloudCluster != null) { NamedList namedList = miniSolrCloudCluster.deleteCollection(MOCK_TABLE); assertEquals(namedList.indexOf("success", 0), 1); Thread.sleep(1000); } } @Test public void testInsert() throws Exception { Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA); assertEquals(Status.OK, result); } @Test public void testDelete() throws Exception { Status result = instance.delete(MOCK_TABLE, MOCK_KEY1); assertEquals(Status.OK, result); } @Test public void testRead() throws Exception { Set fields = MOCK_DATA.keySet(); HashMap resultParam = new HashMap<>(NUM_RECORDS); Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam); assertEquals(Status.OK, result); } @Test public void testUpdate() throws Exception { HashMap newValues = new HashMap<>(NUM_RECORDS); for (int i = 0; i < NUM_RECORDS; i++) { newValues.put("field" + i, new StringByteIterator("newvalue" + i)); } Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues); assertEquals(Status.OK, result); //validate that the values changed HashMap resultParam = new HashMap<>(NUM_RECORDS); instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam); for (int i = 0; i < NUM_RECORDS; i++) { assertEquals("newvalue" + i, resultParam.get("field" + i).toString()); } } @Test public void testScan() throws Exception { Set fields = MOCK_DATA.keySet(); Vector> resultParam = new Vector<>(NUM_RECORDS); Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, NUM_RECORDS, fields, resultParam); assertEquals(Status.OK, result); } /** * Gets the test DB. * * @return The test DB. */ protected DB getDB() { return getDB(new Properties()); } /** * Gets the test DB. * * @param props * Properties to pass to the client. * @return The test DB. */ protected abstract DB getDB(Properties props); } ================================================ FILE: solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientCloudTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr6; import com.yahoo.ycsb.DB; import org.junit.After; import java.util.Properties; import static org.junit.Assume.assumeNoException; public class SolrClientCloudTest extends SolrClientBaseTest { private SolrClient instance; @After public void tearDown() throws Exception { try { if(instance != null) { instance.cleanup(); } } finally { super.tearDown(); } } @Override protected DB getDB(Properties props) { instance = new SolrClient(); props.setProperty("solr.cloud", "true"); props.setProperty("solr.zookeeper.hosts", miniSolrCloudCluster.getSolrClient().getZkHost()); instance.setProperties(props); try { instance.init(); } catch (Exception error) { assumeNoException(error); } return instance; } } ================================================ FILE: solr6/src/test/java/com/yahoo/ycsb/db/solr6/SolrClientTest.java ================================================ /** * Copyright (c) 2016 YCSB contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db.solr6; import com.yahoo.ycsb.DB; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.junit.After; import java.util.Properties; import static org.junit.Assume.assumeNoException; public class SolrClientTest extends SolrClientBaseTest { private SolrClient instance; @After public void tearDown() throws Exception { try { if(instance != null) { instance.cleanup(); } } finally { super.tearDown(); } } @Override protected DB getDB(Properties props) { instance = new SolrClient(); // Use the first Solr server in the cluster. // Doesn't matter if there are more since requests will be forwarded properly by Solr. JettySolrRunner jettySolrRunner = miniSolrCloudCluster.getJettySolrRunners().get(0); String solrBaseUrl = String.format("http://localhost:%s%s", jettySolrRunner.getLocalPort(), jettySolrRunner.getBaseUrl()); props.setProperty("solr.base.url", solrBaseUrl); instance.setProperties(props); try { instance.init(); } catch (Exception error) { assumeNoException(error); } return instance; } } ================================================ FILE: solr6/src/test/resources/log4j.properties ================================================ # Copyright (c) 2015-2016 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # # Root logger option log4j.rootLogger=INFO, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender log4j.appender.stderr.target=System.err log4j.appender.stderr.layout=org.apache.log4j.PatternLayout log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n # Suppress messages from ZooKeeper log4j.logger.org.apache.zookeeper=ERROR # Solr classes are too chatty in test at INFO log4j.logger.org.apache.solr=ERROR log4j.logger.org.eclipse.jetty=ERROR ================================================ FILE: solr6/src/test/resources/solr_config/schema.xml ================================================ text id ================================================ FILE: solr6/src/test/resources/solr_config/solrconfig.xml ================================================ ${tests.luceneMatchVersion:LATEST} ${useCompoundFile:false} ${solr.data.dir:} ${solr.data.dir:} *:* all server-enabled.txt solr ================================================ FILE: tarantool/README.md ================================================ # Tarantool ## Introduction Tarantool is a NoSQL In-Memory database. It's distributed under BSD licence and is hosted on [github][tarantool-github]. Tarantool features: * Defferent index types with iterators: - HASH (the fastest) - TREE (range and ordered retreival) - BITSET (bit mask search) - RTREE (geo search) * multipart keys for HASH and TREE indexes * Data persistence with by Write Ahead Log (WAL) and snapshots. * asynchronous master-master replication, hot standby. * coroutines and async. IO are used to implement high-performance lock-free access to data. - socket-io/file-io with yeilds from lua * stored procedures in Lua (Using LuaJIT) * supports plugins written on C/C++ (Have two basic plugins for working with MySQL and PostgreSQL) * Authentication and access control ## Quick start This section descrives how to run YCSB against a local Tarantool instance ### 1. Start Tarantool First, clone Tarantool from it's own git repo and build it (described in our [README.md][tarantool-readme]): cp %YCSB%/tarantool/conf/tarantool-tree.lua /tarantool.lua cp %TNT%/src/box/tarantool cd ./tarantool tarantool.lua OR you can simply download ans install a binary package for your GNU/Linux or BSD distro from http://tarantool.org/download.html ### 2. Run YCSB Now you are ready to run! First, load the data: ./bin/ycsb load tarantool -s -P workloads/workloada Then, run the workload: ./bin/ycsb run tarantool -s -P workloads/workloada See the next section for the list of configuration parameters for Tarantool. ## Tarantool Configuration Parameters #### 'tarantool.host' (default : 'localhost') Which host YCSB must use for connection with Tarantool #### 'tarantool.port' (default : 3301) Which port YCSB must use for connection with Tarantool #### 'tarantool.space' (default : 1024) (possible values: 0 .. 255) Which space YCSB must use for benchmark Tarantool [tarantool-github]: https://github.com/tarantool/tarantool/ [tarantool-readme]: https://github.com/tarantool/tarantool/blob/master/README.md ================================================ FILE: tarantool/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent/ tarantool-binding Tarantool DB Binding jar org.tarantool connector ${tarantool.version} com.yahoo.ycsb core ${project.version} provided ================================================ FILE: tarantool/src/main/conf/tarantool-hash.lua ================================================ -- Copyright (c) 2015 YCSB contributors. All rights reserved. -- -- Licensed under the Apache License, Version 2.0 (the "License"); you -- may not use this file except in compliance with the License. You -- may obtain a copy of the License at -- -- http://www.apache.org/licenses/LICENSE-2.0 -- -- Unless required by applicable law or agreed to in writing, software -- distributed under the License is distributed on an "AS IS" BASIS, -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -- implied. See the License for the specific language governing -- permissions and limitations under the License. See accompanying -- LICENSE file. box.cfg { listen=3303, logger="tarantool.log", log_level=5, logger_nonblock=true, wal_mode="none", pid_file="tarantool.pid" } box.schema.space.create("ycsb", {id = 1024}) box.space.ycsb:create_index('primary', {type = 'hash', parts = {1, 'STR'}}) box.schema.user.grant('guest', 'read,write,execute', 'universe') ================================================ FILE: tarantool/src/main/conf/tarantool-tree.lua ================================================ -- Copyright (c) 2015 YCSB contributors. All rights reserved. -- -- Licensed under the Apache License, Version 2.0 (the "License"); you -- may not use this file except in compliance with the License. You -- may obtain a copy of the License at -- -- http://www.apache.org/licenses/LICENSE-2.0 -- -- Unless required by applicable law or agreed to in writing, software -- distributed under the License is distributed on an "AS IS" BASIS, -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -- implied. See the License for the specific language governing -- permissions and limitations under the License. See accompanying -- LICENSE file. box.cfg { listen=3303, logger="tarantool.log", log_level=5, logger_nonblock=true, wal_mode="none", pid_file="tarantool.pid" } box.schema.space.create("ycsb", {id = 1024}) box.space.ycsb:create_index('primary', {type = 'tree', parts = {1, 'STR'}}) box.schema.user.grant('guest', 'read,write,execute', 'universe') ================================================ FILE: tarantool/src/main/java/com/yahoo/ycsb/db/TarantoolClient.java ================================================ /** * Copyright (c) 2014 - 2016 YCSB Contributors. All rights reserved. *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import com.yahoo.ycsb.*; import org.tarantool.TarantoolConnection16; import org.tarantool.TarantoolConnection16Impl; import org.tarantool.TarantoolException; import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; /** * YCSB binding for Tarantool. */ public class TarantoolClient extends DB { private static final Logger LOGGER = Logger.getLogger(TarantoolClient.class.getName()); private static final String HOST_PROPERTY = "tarantool.host"; private static final String PORT_PROPERTY = "tarantool.port"; private static final String SPACE_PROPERTY = "tarantool.space"; private static final String DEFAULT_HOST = "localhost"; private static final String DEFAULT_PORT = "3301"; private static final String DEFAULT_SPACE = "1024"; private TarantoolConnection16 connection; private int spaceNo; public void init() throws DBException { Properties props = getProperties(); int port = Integer.parseInt(props.getProperty(PORT_PROPERTY, DEFAULT_PORT)); String host = props.getProperty(HOST_PROPERTY, DEFAULT_HOST); spaceNo = Integer.parseInt(props.getProperty(SPACE_PROPERTY, DEFAULT_SPACE)); try { this.connection = new TarantoolConnection16Impl(host, port); } catch (Exception exc) { throw new DBException("Can't initialize Tarantool connection", exc); } } public void cleanup() throws DBException { this.connection.close(); } @Override public Status insert(String table, String key, Map values) { return replace(key, values, "Can't insert element"); } private HashMap tupleConvertFilter(List input, Set fields) { HashMap result = new HashMap<>(); if (input == null) { return result; } for (int i = 1; i < input.toArray().length; i += 2) { if (fields == null || fields.contains(input.get(i))) { result.put(input.get(i), new StringByteIterator(input.get(i + 1))); } } return result; } @Override public Status read(String table, String key, Set fields, Map result) { try { List response = this.connection.select(this.spaceNo, 0, Arrays.asList(key), 0, 1, 0); result = tupleConvertFilter(response, fields); return Status.OK; } catch (TarantoolException exc) { LOGGER.log(Level.SEVERE, "Can't select element", exc); return Status.ERROR; } catch (NullPointerException exc) { return Status.ERROR; } } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { List> response; try { response = this.connection.select(this.spaceNo, 0, Arrays.asList(startkey), 0, recordcount, 6); } catch (TarantoolException exc) { LOGGER.log(Level.SEVERE, "Can't select range elements", exc); return Status.ERROR; } catch (NullPointerException exc) { return Status.ERROR; } for (List i : response) { HashMap temp = tupleConvertFilter(i, fields); if (!temp.isEmpty()) { result.add((HashMap) temp.clone()); } } return Status.OK; } @Override public Status delete(String table, String key) { try { this.connection.delete(this.spaceNo, Collections.singletonList(key)); } catch (TarantoolException exc) { LOGGER.log(Level.SEVERE, "Can't delete element", exc); return Status.ERROR; } catch (NullPointerException e) { return Status.ERROR; } return Status.OK; } @Override public Status update(String table, String key, Map values) { return replace(key, values, "Can't replace element"); } private Status replace(String key, Map values, String exceptionDescription) { int j = 0; String[] tuple = new String[1 + 2 * values.size()]; tuple[0] = key; for (Map.Entry i : values.entrySet()) { tuple[j + 1] = i.getKey(); tuple[j + 2] = i.getValue().toString(); j += 2; } try { this.connection.replace(this.spaceNo, tuple); } catch (TarantoolException exc) { LOGGER.log(Level.SEVERE, exceptionDescription, exc); return Status.ERROR; } return Status.OK; } } ================================================ FILE: tarantool/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014 - 2016 YCSB Contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ /** * YCSB binding for Tarantool. */ package com.yahoo.ycsb.db; ================================================ FILE: voldemort/pom.xml ================================================ 4.0.0 com.yahoo.ycsb binding-parent 0.14.0-SNAPSHOT ../binding-parent voldemort-binding Voldemort DB Binding jar voldemort voldemort ${voldemort.version} log4j log4j 1.2.16 com.yahoo.ycsb core ${project.version} provided ================================================ FILE: voldemort/src/main/conf/cluster.xml ================================================ mycluster 0 localhost 8081 6666 0, 1 ================================================ FILE: voldemort/src/main/conf/server.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # The ID of *this* particular cluster node node.id=0 max.threads=100 ############### DB options ###################### http.enable=true socket.enable=true # BDB bdb.write.transactions=false bdb.flush.transactions=false bdb.cache.size=1G # Mysql mysql.host=localhost mysql.port=1521 mysql.user=root mysql.password=3306 mysql.database=test #NIO connector settings. enable.nio.connector=true storage.configs=voldemort.store.bdb.BdbStorageConfiguration, voldemort.store.readonly.ReadOnlyStorageConfiguration ================================================ FILE: voldemort/src/main/conf/stores.xml ================================================ usertable bdb client 1 1 1 string java-serialization ================================================ FILE: voldemort/src/main/java/com/yahoo/ycsb/db/VoldemortClient.java ================================================ /** * Copyright (c) 2012 YCSB contributors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. See accompanying * LICENSE file. */ package com.yahoo.ycsb.db; import java.util.HashMap; import java.util.Set; import java.util.Vector; import java.util.Map.Entry; import org.apache.log4j.Logger; import voldemort.client.ClientConfig; import voldemort.client.SocketStoreClientFactory; import voldemort.client.StoreClient; import voldemort.versioning.VectorClock; import voldemort.versioning.Versioned; import com.yahoo.ycsb.DB; import com.yahoo.ycsb.DBException; import com.yahoo.ycsb.Status; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.StringByteIterator; /** * YCSB binding for * Voldemort. */ public class VoldemortClient extends DB { private static final Logger LOGGER = Logger.getLogger(VoldemortClient.class); private StoreClient> storeClient; private SocketStoreClientFactory socketFactory; private String storeName; /** * Initialize the DB layer. This accepts all properties allowed by the * Voldemort client. A store maps to a table. Required : bootstrap_urls * Additional property : store_name -> to preload once, should be same as -t * {@link ClientConfig} */ public void init() throws DBException { ClientConfig clientConfig = new ClientConfig(getProperties()); socketFactory = new SocketStoreClientFactory(clientConfig); // Retrieve store name storeName = getProperties().getProperty("store_name", "usertable"); // Use store name to retrieve client storeClient = socketFactory.getStoreClient(storeName); if (storeClient == null) { throw new DBException("Unable to instantiate store client"); } } public void cleanup() throws DBException { socketFactory.close(); } @Override public Status delete(String table, String key) { if (checkStore(table) == Status.ERROR) { return Status.ERROR; } if (storeClient.delete(key)) { return Status.OK; } return Status.ERROR; } @Override public Status insert(String table, String key, Map values) { if (checkStore(table) == Status.ERROR) { return Status.ERROR; } storeClient.put(key, (HashMap) StringByteIterator.getStringMap(values)); return Status.OK; } @Override public Status read(String table, String key, Set fields, Map result) { if (checkStore(table) == Status.ERROR) { return Status.ERROR; } Versioned> versionedValue = storeClient.get(key); if (versionedValue == null) { return Status.NOT_FOUND; } if (fields != null) { for (String field : fields) { String val = versionedValue.getValue().get(field); if (val != null) { result.put(field, new StringByteIterator(val)); } } } else { StringByteIterator.putAllAsByteIterators(result, versionedValue.getValue()); } return Status.OK; } @Override public Status scan(String table, String startkey, int recordcount, Set fields, Vector> result) { LOGGER.warn("Voldemort does not support Scan semantics"); return Status.OK; } @Override public Status update(String table, String key, Map values) { if (checkStore(table) == Status.ERROR) { return Status.ERROR; } Versioned> versionedValue = storeClient.get(key); HashMap value = new HashMap(); VectorClock version; if (versionedValue != null) { version = ((VectorClock) versionedValue.getVersion()).incremented(0, 1); value = versionedValue.getValue(); for (Entry entry : values.entrySet()) { value.put(entry.getKey(), entry.getValue().toString()); } } else { version = new VectorClock(); StringByteIterator.putAllAsStrings(value, values); } storeClient.put(key, Versioned.value(value, version)); return Status.OK; } private Status checkStore(String table) { if (table.compareTo(storeName) != 0) { try { storeClient = socketFactory.getStoreClient(table); if (storeClient == null) { LOGGER.error("Could not instantiate storeclient for " + table); return Status.ERROR; } storeName = table; } catch (Exception e) { return Status.ERROR; } } return Status.OK; } } ================================================ FILE: voldemort/src/main/java/com/yahoo/ycsb/db/package-info.java ================================================ /* * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. See accompanying LICENSE file. */ /** * The YCSB binding for * Voldemort. */ package com.yahoo.ycsb.db; ================================================ FILE: voldemort/src/main/resources/config/cluster.xml ================================================ mycluster 0 localhost 8081 6666 0, 1 ================================================ FILE: voldemort/src/main/resources/config/server.properties ================================================ # Copyright (c) 2012 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # The ID of *this* particular cluster node node.id=0 max.threads=100 ############### DB options ###################### http.enable=true socket.enable=true # BDB bdb.write.transactions=false bdb.flush.transactions=false bdb.cache.size=1G # Mysql mysql.host=localhost mysql.port=1521 mysql.user=root mysql.password=3306 mysql.database=test #NIO connector settings. enable.nio.connector=true storage.configs=voldemort.store.bdb.BdbStorageConfiguration, voldemort.store.readonly.ReadOnlyStorageConfiguration ================================================ FILE: voldemort/src/main/resources/config/stores.xml ================================================ usertable bdb client 1 1 1 string java-serialization ================================================ FILE: workloads/tsworkload_template ================================================ # Copyright (c) 2017 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Time Series Workload Template: Default Values # # File contains all properties that can be set to define a # YCSB session. All properties are set to their default # value if one exists. If not, the property is commented # out. When a property has a finite number of settings, # the default is enabled and the alternates are shown in # comments below it. # # Use of each property is explained through comments in Client.java, # CoreWorkload.java, TimeSeriesWorkload.java or on the YCSB wiki page: # https://github.com/brianfrankcooper/YCSB/wiki/TimeSeriesWorkload # The name of the workload class to use. Always the following. workload=com.yahoo.ycsb.workloads.TimeSeriesWorkload # The default is Java's Long.MAX_VALUE. # The number of records in the table to be inserted in # the load phase or the number of records already in the # table before the run phase. recordcount=1000000 # There is no default setting for operationcount but it is # required to be set. # The number of operations to use during the run phase. operationcount=3000000 # The number of insertions to do, if different from recordcount. # Used with insertstart to grow an existing table. #insertcount= # ..::NOTE::.. This is different from the CoreWorkload! # The starting timestamp of a run as a Unix Epoch numeral in the # unit set in 'timestampunits'. This is used to determine what # the first timestamp should be when writing or querying as well # as how many offsets (based on 'timestampinterval'). #insertstart= # The units represented by the 'insertstart' timestamp as well as # durations such as 'timestampinterval', 'querytimespan', etc. # For values, see https://docs.oracle.com/javase/7/docs/api/java/util/concurrent/TimeUnit.html # Note that only seconds through nanoseconds are supported. timestampunits=SECONDS # The amount of time between each value in every time series in # the units of 'timestampunits'. timestampinterval=60 # ..::NOTE::.. This is different from the CoreWorkload! # Represents the number of unique "metrics" or "keys" for time series. # E.g. "sys.cpu" may be a single field or "metric" while there may be many # time series sharing that key (perhaps a host tag with "web01" and "web02" # as options). fieldcount=16 # The number of characters in the "metric" or "key". fieldlength=8 # --- TODO ---? # The distribution used to choose the length of a field fieldlengthdistribution=constant #fieldlengthdistribution=uniform #fieldlengthdistribution=zipfian # The number of unique tag combinations for each time series. E.g # if this value is 4, each record will have a key and 4 tag combinations # such as A=A, B=A, C=A, D=A. tagcount=4 # The cardinality (number of unique values) of each tag value for # every "metric" or field as a comma separated list. Each value must # be a number from 1 to Java's Integer.MAX_VALUE and there must be # 'tagcount' values. If there are more or fewer values than #'tagcount' then either it is ignored or 1 is substituted respectively. tagcardinality=1,2,4,8 # The length of each tag key in characters. tagkeylength=8 # The length of each tag value in characters. tagvaluelength=8 # The character separating tag keys from tag values when reads, deletes # or scans are executed against a database. The default is the equals sign # so a field passed in a read to a DB may look like 'AA=AB'. tagpairdelimiter== # The delimiter between keys and tags when a delete is passed to the DB. # E.g. if there was a key and a field, the request key would look like: # 'AA:AA=AB' deletedelimiter=: # Whether or not to randomize the timestamp order when performing inserts # and updates against a DB. By default all writes perform with the # timestamps moving linearly forward in time once all time series for a # given key have been written. randomwritetimestamporder=false # Whether or not to randomly shuffle the time series order when writing. # This will shuffle the keys, tag keys and tag values. # ************************************************************************ # WARNING - When this is enabled, reads and scans will likely return many # empty results as invalid tag combinations will be chosen. Likewise # this setting is INCOMPATIBLE with data integrity checks. # ************************************************************************ randomtimeseriesorder=false # The type of numerical data generated for each data point. The values are # 64 bit signed integers, double precision floating points or a random mix. # For data integrity, this setting is ignored and values are switched to # 64 bit signed ints. #valuetype=integers valuetype=floats #valuetype=mixed # A value from 0 to 0.999999 representing how sparse each time series # should be. The higher this value, the greater the time interval between # values in a single series. For example, if sparsity is 0 and there are # 10 time series with a 'timestampinterval' of 60 seconds with a total # time range of 10 intervals, you would see 100 values written, one per # timestamp interval per time series. If the sparsity is 0.50 then there # would be only about 50 values written so some time series would have # missing values at each interval. sparsity=0.00 # The percentage of time series that are "lagging" behind the current # timestamp of the writer. This is used to mimic a common behavior where # most sources (agents, sensors, etc) are writing data in sync (same timestamp) # but a subset are running behind due to buffering, latency issues, etc. delayedSeries=0.10 # The maximum amount of delay for delayed series in interval counts. The # actual delay is chosen based on a modulo of the series index. delayedIntervals=5 # The fixed or maximum amount of time added to the start time of a # read or scan operation to generate a query over a range of time # instead of a single timestamp. Units are shared with 'timestampunits'. # For example if the value is set to 3600 seconds (1 hour) then # each read would pick a random start timestamp based on the #'insertstart' value and number of intervals, then add 3600 seconds # to create the end time of the query. If this value is 0 then reads # will only provide a single timestamp. # WARNING: Cannot be used with 'dataintegrity'. querytimespan=0 # Whether or not reads should choose a random time span (aligned to # the 'timestampinterval' value) for each read or scan request starting # at 0 and reaching 'querytimespan' as the max. queryrandomtimespan=false # A delimiter character used to separate the start and end timestamps # of a read query when 'querytimespan' is enabled. querytimespandelimiter=, # A unique key given to read, scan and delete operations when the # operation should perform a group-by (multi-series aggregation) on one # or more tags. If 'groupbyfunction' is set, this key will be given with # the configured function. groupbykey=YCSBGB # A function name (e.g. 'sum', 'max' or 'avg') passed during reads, # scans and deletes to cause the database to perform a group-by # operation on one or more tags. If this value is empty or null # (default), group-by operations are not performed #groupbyfunction= # A comma separated list of 0s or 1s to denote which of the tag keys # should be grouped during group-by operations. The number of values # must match the number of tags in 'tagcount'. #groupbykeys=0,0,1,1 # A unique key given to read and scan operations when the operation # should downsample the results of a query into lower resolution # data. If 'downsamplingfunction' is set, this key will be given with # the configured function. downsamplingkey=YCSBDS # A function name (e.g. 'sum', 'max' or 'avg') passed during reads and # scans to cause the database to perform a downsampling operation # returning lower resolution data. If this value is empty or null # (default), downsampling is not performed. #downsamplingfunction= # A time interval for which to downsample the raw data into. Shares # the same units as 'timestampinterval'. This value must be greater # than 'timestampinterval'. E.g. if the timestamp interval for raw # data is 60 seconds, the downsampling interval could be 3600 seconds # to roll up the data into 1 hour buckets. #downsamplinginterval= # What proportion of operations are reads readproportion=0.10 # What proportion of operations are updates updateproportion=0.00 # What proportion of operations are inserts insertproportion=0.90 # The distribution of requests across the keyspace requestdistribution=zipfian #requestdistribution=uniform #requestdistribution=latest # The name of the database table to run queries against table=usertable # Whether or not data should be validated during writes and reads. If # set then the data type is always a 64 bit signed integer and is the # hash code of the key, timestamp and tags. dataintegrity=false # How the latency measurements are presented measurementtype=histogram #measurementtype=timeseries #measurementtype=raw # When measurementtype is set to raw, measurements will be output # as RAW datapoints in the following csv format: # "operation, timestamp of the measurement, latency in us" # # Raw datapoints are collected in-memory while the test is running. Each # data point consumes about 50 bytes (including java object overhead). # For a typical run of 1 million to 10 million operations, this should # fit into memory most of the time. If you plan to do 100s of millions of # operations per run, consider provisioning a machine with larger RAM when using # the RAW measurement type, or split the run into multiple runs. # # Optionally, you can specify an output file to save raw datapoints. # Otherwise, raw datapoints will be written to stdout. # The output file will be appended to if it already exists, otherwise # a new output file will be created. #measurement.raw.output_file = /tmp/your_output_file_for_this_run # JVM Reporting. # # Measure JVM information over time including GC counts, max and min memory # used, max and min thread counts, max and min system load and others. This # setting must be enabled in conjunction with the "-s" flag to run the status # thread. Every "status.interval", the status thread will capture JVM # statistics and record the results. At the end of the run, max and mins will # be recorded. # measurement.trackjvm = false # The range of latencies to track in the histogram (milliseconds) histogram.buckets=1000 # Granularity for time series (in milliseconds) timeseries.granularity=1000 # Latency reporting. # # YCSB records latency of failed operations separately from successful ones. # Latency of all OK operations will be reported under their operation name, # such as [READ], [UPDATE], etc. # # For failed operations: # By default we don't track latency numbers of specific error status. # We just report latency of all failed operation under one measurement name # such as [READ-FAILED]. But optionally, user can configure to have either: # 1. Record and report latency for each and every error status code by # setting reportLatencyForEachError to true, or # 2. Record and report latency for a select set of error status codes by # providing a CSV list of Status codes via the "latencytrackederrors" # property. # reportlatencyforeacherror=false # latencytrackederrors="" ================================================ FILE: workloads/tsworkloada ================================================ # Copyright (c) 2017 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload A: Small cardinality consistent data for 2 days # Application example: Typical monitoring of a single compute or small # sensor station where 90% of the load is write and only 10% is read # (it's usually much less). All writes are inserts. No sparsity so # every series will have a value at every timestamp. # # Read/insert ratio: 10/90 # Cardinality: 16 per key (field), 64 fields for a total of 1,024 # time series. workload=com.yahoo.ycsb.workloads.TimeSeriesWorkload recordcount=1474560 operationcount=2949120 fieldlength=8 fieldcount=64 tagcount=4 tagcardinality=1,2,4,2 sparsity=0.0 delayedSeries=0.0 delayedIntervals=0 timestampunits=SECONDS timestampinterval=60 querytimespan=3600 readproportion=0.10 updateproportion=0.00 insertproportion=0.90 ================================================ FILE: workloads/workload_template ================================================ # Copyright (c) 2012-2016 YCSB contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload Template: Default Values # # File contains all properties that can be set to define a # YCSB session. All properties are set to their default # value if one exists. If not, the property is commented # out. When a property has a finite number of settings, # the default is enabled and the alternates are shown in # comments below it. # # Use of most explained through comments in Client.java or # CoreWorkload.java or on the YCSB wiki page: # https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties # The name of the workload class to use workload=com.yahoo.ycsb.workloads.CoreWorkload # There is no default setting for recordcount but it is # required to be set. # The number of records in the table to be inserted in # the load phase or the number of records already in the # table before the run phase. recordcount=1000000 # There is no default setting for operationcount but it is # required to be set. # The number of operations to use during the run phase. operationcount=3000000 # The number of insertions to do, if different from recordcount. # Used with insertstart to grow an existing table. #insertcount= # The offset of the first insertion insertstart=0 # The number of fields in a record fieldcount=10 # The size of each field (in bytes) fieldlength=100 # Should read all fields readallfields=true # Should write all fields on update writeallfields=false # The distribution used to choose the length of a field fieldlengthdistribution=constant #fieldlengthdistribution=uniform #fieldlengthdistribution=zipfian # What proportion of operations are reads readproportion=0.95 # What proportion of operations are updates updateproportion=0.05 # What proportion of operations are inserts insertproportion=0 # What proportion of operations read then modify a record readmodifywriteproportion=0 # What proportion of operations are scans scanproportion=0 # On a single scan, the maximum number of records to access maxscanlength=1000 # The distribution used to choose the number of records to access on a scan scanlengthdistribution=uniform #scanlengthdistribution=zipfian # Should records be inserted in order or pseudo-randomly insertorder=hashed #insertorder=ordered # The distribution of requests across the keyspace requestdistribution=zipfian #requestdistribution=uniform #requestdistribution=latest # Percentage of data items that constitute the hot set hotspotdatafraction=0.2 # Percentage of operations that access the hot set hotspotopnfraction=0.8 # Maximum execution time in seconds #maxexecutiontime= # The name of the database table to run queries against table=usertable # The column family of fields (required by some databases) #columnfamily= # How the latency measurements are presented measurementtype=histogram #measurementtype=timeseries #measurementtype=raw # When measurementtype is set to raw, measurements will be output # as RAW datapoints in the following csv format: # "operation, timestamp of the measurement, latency in us" # # Raw datapoints are collected in-memory while the test is running. Each # data point consumes about 50 bytes (including java object overhead). # For a typical run of 1 million to 10 million operations, this should # fit into memory most of the time. If you plan to do 100s of millions of # operations per run, consider provisioning a machine with larger RAM when using # the RAW measurement type, or split the run into multiple runs. # # Optionally, you can specify an output file to save raw datapoints. # Otherwise, raw datapoints will be written to stdout. # The output file will be appended to if it already exists, otherwise # a new output file will be created. #measurement.raw.output_file = /tmp/your_output_file_for_this_run # JVM Reporting. # # Measure JVM information over time including GC counts, max and min memory # used, max and min thread counts, max and min system load and others. This # setting must be enabled in conjunction with the "-s" flag to run the status # thread. Every "status.interval", the status thread will capture JVM # statistics and record the results. At the end of the run, max and mins will # be recorded. # measurement.trackjvm = false # The range of latencies to track in the histogram (milliseconds) histogram.buckets=1000 # Granularity for time series (in milliseconds) timeseries.granularity=1000 # Latency reporting. # # YCSB records latency of failed operations separately from successful ones. # Latency of all OK operations will be reported under their operation name, # such as [READ], [UPDATE], etc. # # For failed operations: # By default we don't track latency numbers of specific error status. # We just report latency of all failed operation under one measurement name # such as [READ-FAILED]. But optionally, user can configure to have either: # 1. Record and report latency for each and every error status code by # setting reportLatencyForEachError to true, or # 2. Record and report latency for a select set of error status codes by # providing a CSV list of Status codes via the "latencytrackederrors" # property. # reportlatencyforeacherror=false # latencytrackederrors="" # Insertion error retry for the core workload. # # By default, the YCSB core workload does not retry any operations. # However, during the load process, if any insertion fails, the entire # load process is terminated. # If a user desires to have more robust behavior during this phase, they can # enable retry for insertion by setting the following property to a positive # number. # core_workload_insertion_retry_limit = 0 # # the following number controls the interval between retries (in seconds): # core_workload_insertion_retry_interval = 3 # Distributed Tracing via Apache HTrace (http://htrace.incubator.apache.org/) # # Defaults to blank / no tracing # Below sends to a local file, sampling at 0.1% # # htrace.sampler.classes=ProbabilitySampler # htrace.sampler.fraction=0.001 # htrace.span.receiver.classes=org.apache.htrace.core.LocalFileSpanReceiver # htrace.local.file.span.receiver.path=/some/path/to/local/file # # To capture all spans, use the AlwaysSampler # # htrace.sampler.classes=AlwaysSampler # # To send spans to an HTraced receiver, use the below and ensure # your classpath contains the htrace-htraced jar (i.e. when invoking the ycsb # command add -cp /path/to/htrace-htraced.jar) # # htrace.span.receiver.classes=org.apache.htrace.impl.HTracedSpanReceiver # htrace.htraced.receiver.address=example.com:9075 # htrace.htraced.error.log.period.ms=10000 ================================================ FILE: workloads/workloada ================================================ # Copyright (c) 2010 Yahoo! Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload A: Update heavy workload # Application example: Session store recording recent actions # # Read/update ratio: 50/50 # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) # Request distribution: zipfian recordcount=1000 operationcount=1000 workload=com.yahoo.ycsb.workloads.CoreWorkload readallfields=true readproportion=0.5 updateproportion=0.5 scanproportion=0 insertproportion=0 requestdistribution=zipfian ================================================ FILE: workloads/workloadb ================================================ # Copyright (c) 2010 Yahoo! Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload B: Read mostly workload # Application example: photo tagging; add a tag is an update, but most operations are to read tags # # Read/update ratio: 95/5 # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) # Request distribution: zipfian recordcount=1000 operationcount=1000 workload=com.yahoo.ycsb.workloads.CoreWorkload readallfields=true readproportion=0.95 updateproportion=0.05 scanproportion=0 insertproportion=0 requestdistribution=zipfian ================================================ FILE: workloads/workloadc ================================================ # Copyright (c) 2010 Yahoo! Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload C: Read only # Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop) # # Read/update ratio: 100/0 # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) # Request distribution: zipfian recordcount=1000 operationcount=1000 workload=com.yahoo.ycsb.workloads.CoreWorkload readallfields=true readproportion=1 updateproportion=0 scanproportion=0 insertproportion=0 requestdistribution=zipfian ================================================ FILE: workloads/workloadd ================================================ # Copyright (c) 2010 Yahoo! Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload D: Read latest workload # Application example: user status updates; people want to read the latest # # Read/update/insert ratio: 95/0/5 # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) # Request distribution: latest # The insert order for this is hashed, not ordered. The "latest" items may be # scattered around the keyspace if they are keyed by userid.timestamp. A workload # which orders items purely by time, and demands the latest, is very different than # workload here (which we believe is more typical of how people build systems.) recordcount=1000 operationcount=1000 workload=com.yahoo.ycsb.workloads.CoreWorkload readallfields=true readproportion=0.95 updateproportion=0 scanproportion=0 insertproportion=0.05 requestdistribution=latest ================================================ FILE: workloads/workloade ================================================ # Copyright (c) 2010 Yahoo! Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload E: Short ranges # Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id) # # Scan/insert ratio: 95/5 # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) # Request distribution: zipfian # The insert order is hashed, not ordered. Although the scans are ordered, it does not necessarily # follow that the data is inserted in order. For example, posts for thread 342 may not be inserted contiguously, but # instead interspersed with posts from lots of other threads. The way the YCSB client works is that it will pick a start # key, and then request a number of records; this works fine even for hashed insertion. recordcount=1000 operationcount=1000 workload=com.yahoo.ycsb.workloads.CoreWorkload readallfields=true readproportion=0 updateproportion=0 scanproportion=0.95 insertproportion=0.05 requestdistribution=zipfian maxscanlength=100 scanlengthdistribution=uniform ================================================ FILE: workloads/workloadf ================================================ # Copyright (c) 2010 Yahoo! Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. See accompanying # LICENSE file. # Yahoo! Cloud System Benchmark # Workload F: Read-modify-write workload # Application example: user database, where user records are read and modified by the user or to record user activity. # # Read/read-modify-write ratio: 50/50 # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) # Request distribution: zipfian recordcount=1000 operationcount=1000 workload=com.yahoo.ycsb.workloads.CoreWorkload readallfields=true readproportion=0.5 updateproportion=0 scanproportion=0 insertproportion=0 readmodifywriteproportion=0.5 requestdistribution=zipfian