[ { "path": ".gitignore", "content": "build/\n.gradle/\n*~\n*.jar\ncassandra-loader\n*.BADINSERT\n*.BADPARSE\n*.LOG\n" }, { "path": "ChangeLog.md", "content": "## 0.0.27\n- Added support for specifying a TTL (Issue 67)\n\n## 0.0.26\n- Fixed issue with long schemas (Issue 65)\n\n## 0.0.25\n- Added support for Fetch Size in cassandra-unloader\n\n## 0.0.24\n- Added support for DATE (and associated -localDateFormat option)\n- Added support for SHORTINT and TINYINT\n- Fixed an issue with special characters in column names (Issue 59)\n\n## 0.0.23\n- Fixed case of commas in Map keys\n- Fixed keystore/truststore-pw typo (was pwd)\n- Fixed quoting of non-collections\n- Added support for supplying comment character\n\n## 0.0.22\n- Added support for gzipped input files\n- Fixed issue with quoted values (and in collections)\n\n## 0.0.21\n- Added support for jsonarray (one JSON array per file)\n- Added support for jsonline (one JSON per line)\n- Changed JVM settings to 1GB heap\n- Added -charsPerColumn (default to 4096) to enable Univocity optimizations\n- Removed explicit queries to metadata tables and use driver API calls\n\n## 0.0.20\n- Fixed delimiter in MapParser\n- Catch NULLs in Map/Set/List parsing and throw in BADPARSE\n\n## 0.0.19\n- Converted to parse with Univocity CSV parser\n- Added -where\n\n## 0.0.18\n- Support for Cassandra 3.0\n\n## 0.0.17\n- Fixed null collection issue / NPE (Issue 8)\n\n## 0.0.16\n- Unloader will quote collections (which the loader expects)\n- Fixed collection issue (Issue 14)\n- BLOBs are now Base64 encoded on unload, and should be Base64 to load (Issue 15)\n- Support for quoted keyspace, table, and column names\n\n## 0.0.15\n- Better error handling for case when C* inserts are failing\n\n## 0.0.14\n- Updated cassandra-unloader to add support for collections,\n\tconsistency level, ssl, etc\n\n## 0.0.13\n- Added configFile\n- added ssl options (with truststore and keystore)\n\n## 0.0.12\n- Added a rateFile to output CSV rate statistics\n- added -skipCols to skip input columns\n\n## 0.0.11\n- Added support for quoted Keyspaces, Tables, and Columns\n\n## 0.0.10\n- You want collections? You got 'em\n- Added progress reporting - you can specify the rate at which\n\tthe rate is reported via the -progressRate option\n- Refactored RateLimiting - added it to a new RateLimitingSession\n- Laid groundwork for Dynamic rate limiting - to be worked out\n\tonce we find a way to collect the right statistic\n\n## 0.0.9\n- Added -successDir and -failureDir\n- Added return codes for the loader and unloader\n- Refactored BoolStyle\n- Cleaned up the readme a bit\n\n" }, { "path": "LICENSE.md", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2016, DataStax\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "\n# CASSANDRA-LOADER\n\n## Introduction\n\ncassandra-loader is a general-purpose, delimited-file, bulk loader for \nCassandra. It supports a number of configuration options to enable bulk \nloading of various types of delimited files, including\n* comma-separated values\n* tab-separated values\n* customer delimiter-separated values\n* header row\n* comma as decimal separator\n* ...\n\n## Getting it\n\n### Downloading\n\nThis utility has already been built, and is available at\nhttps://github.com/brianmhess/cassandra-loader/releases/download/v0.0.27/cassandra-loader\n\nGet it with wget:\n\n```\nwget https://github.com/brianmhess/cassandra-loader/releases/download/v0.0.27/cassandra-loader\n```\n\n### Building\n\nTo build this repository, simply clone this repo and run:\n\n```\ngradle loader\n```\n\nAll of the dependencies are included (namely, the Java driver - currently\nversion 3.0.0). The output will be the cassandra-loader executable\nin the build directory. There will also be an jar with all of the\ndependencies included in the build/libs/cassandra-loader-uber-.jar\n\n## Documentation \n\nTo extract this README document, simply run (on the cassandra-loader\nexecutable - (e.g., on build/cassandra-loader):\n\n```\njar xf cassandra-loader README.md\n```\n\n## Run\n\nTo run cassandra-loader, simply run the cassandra-loader executable \n(e.g., located at build/cassandra-loader):\n\n```\ncassandra-loader\n```\n\nIf you built this with gradle, you can also run:\n\n```\ngradle run\n```\n\nThis will print the usage statement.\n\nThe following will load the `myFileToLoad.csv` file into the Cassandra \ncluster at IP address 1.2.3.4 into the `test.ltest` column family where \nthe myFileToLoad file has the format of 4 columns - and it gets the\ndata type information from the database - and using the default options:\n\n```\ncassandra-loader -f myFileToLoad.csv -host 1.2.3.4 -schema \"test.ltest(a, b, c, d)\"\n```\n\n## Options:\n\n Switch | Option | Default | Description\n-----------------:|-------------------:|---------------------------:|:----------\n `-configFile` | Filename | none | Filename of configuration options \n `-f` | Filename | <REQUIRED> | Filename to load - required.\n `-host` | IP Address | <REQUIRED> | Cassandra connection point - required.\n `-format` | Input format | delim | Format of the data. Options are \"delim\" or \"json\".\n `-schema` | CQL schema | | Schema of input data - required for delim In the format \"keySpace.table(col1,col2,...)\" and in the order that the data will be in the file.\n `-keyspace` | Keyspace name | | Name of keyspace (case sensitive) to load in to - required for json\n `-table` | Table name | | Name of table (case sensitive) to load in to - required for json\n `-port` | Port Number | 9042 | Cassandra native protocol port number\n `-user` | Username | none | Cassandra username\n `-pw` | Password | none | Cassandra password\n `-ssl-truststore-path` | Truststore Path | none | Path to SSL truststore\n `-ssl-truststore-pw` | Truststore Password | none | Password to SSL truststore\n `-ssl-keystore-path` | Keystore Path | none | Path to SSL keystore\n `-ssl-keystore-path` | Keystore Password | none | Password to SSL keystore\n `-consistencyLevel` | Consistency Level | ONE | CQL Consistency Level\n `-numThreads` | Number of threads | Number of CPUs | Number of threads to use (one per file)\n `-numFutures` | Number of Futures | 1000 | Number of Java driver futures in flight.\n `-numRetries` | Number of retries | 1 | Number of times to retry the INSERT before declaring defeat.\n `-queryTimeout` | Timeout in seconds | 2 | Amount of time to wait for a query to finish before timing out.\n `-ttl` | Time To Live | none | TTL to use when inserting these rows\n `-delim` | Delimiter | , | Delimiter to use\n `-charsPerColumn`| Characters per column | 4096 | Maximum characters per column\n `-nullString` | Null String | <empty string> | String to represent NULL data\n `-boolStyle` | Boolean Style | TRUE_FALSE | String for boolean values. Options are \"1_0\", \"Y_N\", \"T_F\", \"YES_NO\", \"TRUE_FALSE\".\n `-decimalDelim` | Decimal delimiter | . | Delimiter for decimal values. Options are \".\" or \",\"\n `-dateFormat` | Date Format String | default for Locale.ENGLISH | Date format string as specified in the SimpleDateFormat Java class: http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html\n `-skipRows` | Rows to skip | 0 | Number of rows to skip at the beginning of the file\n `-skipCols` | Columns to skip | <not set> | Comma-separated list of columns to skip loading (0-counted)\n `-maxRows` | Max rows to read | -1 | Maximum rows to read (after optional skipping of rows). -1 signifies all rows.\n `-maxErrors` | Max parse errors | 10 | Maximum number of rows that do not parse to allow before exiting.\n `-maxInsertErrors`| Max insert errors | 10 | Maximum number of rows that do not insert to allow before exiting.\n `-badDir` | Bad directory | current directory | Directory to write badly parsed and badly inserted rows - as well as the log file.\n `-rate` | Ingest rate | unlimited | Maximum rate to insert data - in rows/sec.\n `-progressRate` | Progress rate | 100000 | How often to report the ingest rate (number of rows)\n `-rateFile` | Rate Stats File | <not set> | File to contain CSV rate statistics\n `-successDir` | Success directory | <not set> | Location to move successfully loaded files\n `-failureDir` | Failure directory | <not set> | Location to move files that failed to load\n `-batchSize` | Batch size | 1 | Size of unlogged batches. If set to 1 then no batching.\n `-comment` | Comment character | $lt;not set> | Comment character.\n\n## Comments\n\n### Using stdin\n\nYou can send data in on stdin by specifying the filename (via the -f switch) as \"stdin\" (case insensitive).\nThat way, you could pipe data in from other commands:\n\n```\ngrep IMPORTANT data.csv | cassandra-loader -f stdin -h 1.2.3.4 -schema \"test.itest(a, b)\"\n```\n\n### Support for collections \n\nCollections are supported. Their format is the CQL native one.\nSets are started with '{' and ended with '}' and enclose a comma-separated list\n{1,2,3} or {\"a\",\"b\",\"c\"}\nLists are started with '[' and ended with ']' and enclose a comma-separated list\n[1,2,3] or [\"a\",\"b\",\"c\"]\nMaps are started with '{' and ended with '}' and enclose a comma-separated list\nof pairs that are separated by ':'\n{1:1,2:2,3:3} or {\"a\":1, \"b\":2, \"c\":3}\nAll collections must be enclosed in double-quotes.\n\n### Username/Password\n\nIf you specify either the username or the password, then you must specify both.\n\n### Boolean Style\n\nboolStyle is a case-insensitive test of the True and False strings. For the\ndifferent styles, the True and False strings are as follows:\n\n```\n Style | True | False\n------------|------|-------\n 0_1 | 1 | 0 \n Y_N | Y | N \n T_F | T | F \n YES_NO | YES | NO \n TRUE_FALSE | TRUE | FALSE \n```\n\n### Configuration file\n\nconfigFile is a file with configuration options that are formatted just like on\nthe command line. This allows you to not specify arguments on the command line.\nFor example, you can specify passwords in the configFile and avoid having them on\nthe command line. The format is one switch and option per line:\n\n```\n-pwd mypassword\n-host 1.2.3.4\n```\n\n### Miscelaneous\n\nnumFutures is a way to control the level of parallelism, but at some point \ntoo many will actually slow down the load. The default of 500 is a decent \nplace to start.\n\nIf you use quoted strings, you need to use double-quotes. To escape a double-quote inside a quoted string, use the backslash to escape it (\"\\\"\"). To create a backslash inside a quoted string, use two backslashes in a row (\"\\\\\"). If you quote your string, it will not be trimmed, but if you do not quote your string it will be trimmed.\n\nLoading into counter columns is not supported.\n\nThe default nullString is the empty string. If you want empty strings to be saved as empty strings, set the nullString to something else.\n\nIf you do not set the successDir then files that successfully loaded will remain in their input directory. The same is true for failed files if you do not set the failureDir. You cannot set either if the input file is \"stdin\".\n\nWhen using `jsonline`, all JSON field names are case-sensitive. When using `jsonline` or `jsonarray`, the `-keyspace` and `-table` arguments are case-sensitive.\n\n## Usage Statement:\n\n```\nversion: 0.0.27\nUsage: -f -host [OPTIONS]\nOPTIONS:\n -schema Table schema (when using delim)\n -table Table name (when using json)\n -keyspace Keyspace name (when using json)\n -configFile File with configuration options\n -delim Delimiter to use [,]\n -charsPerColumn Max number of chars per column [4096]\n -dateFormat Date format for TIMESTAMP [default for Locale.ENGLISH]\n -localDateFormat Date format for DATE [yyyy-MM-dd]\n -nullString String that signifies NULL [none]\n -comment Comment symbol to use [none]\n -skipRows Number of rows to skip [0]\n -skipCols Comma-separated list of columsn to skip in the input file\n -maxRows Maximum number of rows to read (-1 means all) [-1]\n -maxErrors Maximum parse errors to endure [10]\n -badDir Directory for where to place badly parsed rows. [none]\n -port CQL Port Number [9042]\n -user Cassandra username [none]\n -pw Password for user [none]\n -ssl-truststore-path Path to SSL truststore [none]\n -ssl-truststore-pw Password for SSL truststore [none]\n -ssl-keystore-path Path to SSL keystore [none]\n -ssl-keystore-pw Password for SSL keystore [none]\n -consistencyLevel Consistency level [LOCAL_ONE]\n -numFutures Number of CQL futures to keep in flight [1000]\n -batchSize Number of INSERTs to batch together [1]\n -decimalDelim Decimal delimiter [.] Other option is ','\n -boolStyle Style for booleans [TRUE_FALSE]\n -numThreads Number of concurrent threads (files) to load [num cores]\n -queryTimeout <# seconds> Query timeout (in seconds) [2]\n -numRetries Number of times to retry the INSERT [1]\n -maxInsertErrors <# errors> Maximum INSERT errors to endure [10]\n -rate Maximum insert rate [50000]\n -progressRate How often to report the insert rate [100000]\n -rateFile Where to print the rate statistics\n -successDir Directory where to move successfully loaded files\n -failureDir Directory where to move files that did not successfully load\n -nullsUnset [false|true] Treat nulls as unset [faslse]\n -format [delim|jsonline|jsonarray] Format of data: delimited or JSON [delim]\n -table Table name (when using JSON)\n -keyspace Keyspace name (when using JSON)\n -ttl TTL for all rows in this invocation [unset]\n\n\nExamples:\ncassandra-loader -f /path/to/file.csv -host localhost -schema \"test.test3(a, b, c)\"\ncassandra-loader -f /path/to/directory -host 1.2.3.4 -schema \"test.test3(a, b, c)\" -delim \"\\t\" -numThreads 10\ncassandra-loader -f stdin -host localhost -schema \"test.test3(a, b, c)\" -user myuser -pw mypassword\n```\n\n##Examples:\n\nLoad file /path/to/file.csv into the test3 table in the test keyspace using\nthe cluster at localhost. Use the default options:\n\n```\ncassandra-loader -f /path/to/file.csv -host localhost -schema \"test.test3(a, b, c)\"\n```\n\nLoad all the files from /path/to/directory into the test3 table in the test\nkeyspace using the cluster at 1.2.3.4. Use 10 threads and use tab as the\ndelimiter:\n\n```\ncassandra-loader -f /path/to/directory -host 1.2.3.4 -schema \"test.test3(a, b, c)\" -delim \"\\t\" -numThreads 10\n```\n\nLoad the data from stdin into the test3 table in the test keyspace using the\ncluster at localhost. Use \"myuser\" as the username and \"mypassword\" as the\npassword:\n\n```\ncassandra-loader -f stdin -host localhost -schema \"test.test3(a, b, c)\" -user myuser -pw mypassword\n```\n\n##Sample\n\nIncluded here is a set of sample data. It is in the sample/ directory.\nYou can set up the table and keyspace by running:\n\n```\ncqlsh -f sample/cassandra-schema.cql\n```\n\nTo load the data, run:\n\n```\ncd sample\n./load.sh\n```\n\nTo check that things have succeeded, you can run:\n\n```\nwc -l titanic.csv\n```\n\nAnd:\n\n```\ncqlsh -e \"SELECT COUNT(*) FROM titanic.surviors\"\n```\n\nBoth should return 891.\n\n\n\n## cassandra-unloader\ncassandra-unloader is a utility to dump the contents\nof a Cassandra table to delimited file format. It uses\nthe same sorts of options as cassandra-loader so that the\noutput of cassandra-unloader could be piped into \ncassandra-loader:\n\n```\ncassandra-unloader -f stdout -host host1 -schema \"ks.table(a,b,c)\" | cassandra-loader -f stdin -host host2 -schema \"ks2.table2(x,y,z)\"\n```\n\nGet it with wget:\n```\nwget https://github.com/brianmhess/cassandra-loader/releases/download/v0.0.27/cassandra-unloader\n```\n\nTo build, run:\n\n```\ngradle unloader\n```\n\nTo run cassandra-unloader, simply run the cassandra-unloader executable \n(e.g., located at build/cassandra-unloader):\n\n```\ncassandra-unloader\n```\n\n###Usage statement:\n\n```\nversion: 0.0.27\nUsage: -f -host -schema [OPTIONS]\nOPTIONS:\n -configFile File with configuration options\n -format [delim|jsonline|jsonarray] Format of data: delimited or JSON [delim]\n -delim Delimiter to use [,]\n -dateFormat Date format for TIMESTAMP [default for Locale.ENGLISH]\n -localDateFormat Date format for DATE [yyyy-MM-dd]\n -nullString String that signifies NULL [none]\n -port CQL Port Number [9042]\n -user Cassandra username [none]\n -pw Password for user [none]\n -ssl-truststore-path Path to SSL truststore [none]\n -ssl-truststore-pw Password for SSL truststore [none]\n -ssl-keystore-path Path to SSL keystore [none]\n -ssl-keystore-pw Password for SSL keystore [none]\n -consistencyLevel Consistency level [LOCAL_ONE]\n -decimalDelim Decimal delimiter [.] Other option is ','\n -boolStyle Style for booleans [TRUE_FALSE]\n -numThreads Number of concurrent threads to unload [5]\n -beginToken Begin token [none]\n -endToken End token [none]\n -where WHERE clause [none]\n -fetchSize Fetch size to use [0]\n```\n\nA few simple examples using the `-where` are as follows:\n\n```\ncassandra-unloader -host localhost -f stdout -schema \"testks.testtable(pkey,ccol,x,y)\" -where \"pkey=5\"\ncassandra-unloader -host localhost -f stdout -schema \"testks.testtable(pkey,ccol,x,y)\" -where \"x = 100 ALLOW FILTERING\"\n```\n" }, { "path": "build.gradle", "content": "apply plugin: 'java'\napply plugin: 'application'\n\ndef versionNum = '0.0.27'\n\nallprojects {\n tasks.withType(JavaCompile) {\n options.compilerArgs << \"-Xlint:unchecked\"\n options.compilerArgs << \"-Xlint:deprecation\"\n }\n}\n\ntask loader(type: Exec) {\n dependsOn << 'uberloader'\n commandLine './src/make/buildit.sh'\n}\n\ntask unloader(type: Exec) {\n dependsOn << 'uberunloader'\n commandLine './src/make/unloader.sh'\n}\n\njar {\n baseName = 'cassandra-loader'\n version = versionNum\n}\n\nrepositories {\n mavenCentral()\n}\n\ndependencies {\n compile 'com.datastax.cassandra:cassandra-driver-core:3.2.0'\n compile 'org.xerial.snappy:snappy-java:1.0.5'\n compile 'net.jpountz.lz4:lz4:1.2.0'\n compile 'ch.qos.logback:logback-classic:1.1.3'\n compile 'com.univocity:univocity-parsers:2.2.3'\n compile 'org.apache.commons:commons-lang3:3.0'\n compile group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1'\n}\n\ntask uberloader(type: Jar) {\n dependsOn << 'compileJava'\n from \"src/main/resources/logback.xml\"\n from files(sourceSets.main.output.classesDir)\n from(configurations.runtime.asFileTree.files.collect { zipTree(it) }) {\n exclude \"META-INF/**\"\n }\n manifest {\n attributes \"Main-Class\" : 'com.datastax.loader.CqlDelimLoad'\n }\n baseName = 'cassandra-loader-uber'\n version = versionNum\n}\n\ntask uberunloader(type: Jar) {\n dependsOn << 'compileJava'\n from \"src/main/resources/logback.xml\"\n from files(sourceSets.main.output.classesDir)\n from(configurations.runtime.asFileTree.files.collect { zipTree(it) }) {\n exclude \"META-INF/**\"\n }\n manifest {\n attributes \"Main-Class\" : 'com.datastax.loader.CqlDelimUnload'\n }\n baseName = 'cassandra-unloader-uber'\n version = versionNum\n}\n" }, { "path": "sample/cassandra-schema.cql", "content": "create keyspace titanic WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};\n\nuse titanic;\n\nCREATE TABLE survivors (\n id varchar,\n survived boolean,\n passenger_class int,\n name varchar,\n sex varchar,\n age int,\n num_siblings_spouse int,\n num_parents_children int,\n ticket_id varchar,\n fare varchar,\n cabin varchar,\n port_of_embarkation varchar,\n primary key (id));" }, { "path": "sample/load.sh", "content": "../build/cassandra-loader -f titanic.csv -host localhost -schema \"titanic.survivors(id, survived, passenger_class, name, sex, age, num_siblings_spouse, num_parents_children, ticket_id, fare, cabin, port_of_embarkation)\" -boolStyle 1_0\n" }, { "path": "sample/titanic.csv", "content": "1,0,3,\"Braund, Mr. Owen Harris\",male,22,1,0,A/5 21171,7.25,,S\n2,1,1,\"Cumings, Mrs. John Bradley (Florence Briggs Thayer)\",female,38,1,0,PC 17599,71.2833,C85,C\n3,1,3,\"Heikkinen, Miss. Laina\",female,26,0,0,STON/O2. 3101282,7.925,,S\n4,1,1,\"Futrelle, Mrs. Jacques Heath (Lily May Peel)\",female,35,1,0,113803,53.1,C123,S\n5,0,3,\"Allen, Mr. William Henry\",male,35,0,0,373450,8.05,,S\n6,0,3,\"Moran, Mr. James\",male,,0,0,330877,8.4583,,Q\n7,0,1,\"McCarthy, Mr. Timothy J\",male,54,0,0,17463,51.8625,E46,S\n8,0,3,\"Palsson, Master. Gosta Leonard\",male,2,3,1,349909,21.075,,S\n9,1,3,\"Johnson, Mrs. Oscar W (Elisabeth Vilhelmina Berg)\",female,27,0,2,347742,11.1333,,S\n10,1,2,\"Nasser, Mrs. Nicholas (Adele Achem)\",female,14,1,0,237736,30.0708,,C\n11,1,3,\"Sandstrom, Miss. Marguerite Rut\",female,4,1,1,PP 9549,16.7,G6,S\n12,1,1,\"Bonnell, Miss. Elizabeth\",female,58,0,0,113783,26.55,C103,S\n13,0,3,\"Saundercock, Mr. William Henry\",male,20,0,0,A/5. 2151,8.05,,S\n14,0,3,\"Andersson, Mr. Anders Johan\",male,39,1,5,347082,31.275,,S\n15,0,3,\"Vestrom, Miss. Hulda Amanda Adolfina\",female,14,0,0,350406,7.8542,,S\n16,1,2,\"Hewlett, Mrs. (Mary D Kingcome) \",female,55,0,0,248706,16,,S\n17,0,3,\"Rice, Master. Eugene\",male,2,4,1,382652,29.125,,Q\n18,1,2,\"Williams, Mr. Charles Eugene\",male,,0,0,244373,13,,S\n19,0,3,\"Vander Planke, Mrs. Julius (Emelia Maria Vandemoortele)\",female,31,1,0,345763,18,,S\n20,1,3,\"Masselmani, Mrs. Fatima\",female,,0,0,2649,7.225,,C\n21,0,2,\"Fynney, Mr. Joseph J\",male,35,0,0,239865,26,,S\n22,1,2,\"Beesley, Mr. Lawrence\",male,34,0,0,248698,13,D56,S\n23,1,3,\"McGowan, Miss. Anna \"\"Annie\"\"\",female,15,0,0,330923,8.0292,,Q\n24,1,1,\"Sloper, Mr. William Thompson\",male,28,0,0,113788,35.5,A6,S\n25,0,3,\"Palsson, Miss. Torborg Danira\",female,8,3,1,349909,21.075,,S\n26,1,3,\"Asplund, Mrs. Carl Oscar (Selma Augusta Emilia Johansson)\",female,38,1,5,347077,31.3875,,S\n27,0,3,\"Emir, Mr. Farred Chehab\",male,,0,0,2631,7.225,,C\n28,0,1,\"Fortune, Mr. Charles Alexander\",male,19,3,2,19950,263,C23 C25 C27,S\n29,1,3,\"O'Dwyer, Miss. Ellen \"\"Nellie\"\"\",female,,0,0,330959,7.8792,,Q\n30,0,3,\"Todoroff, Mr. Lalio\",male,,0,0,349216,7.8958,,S\n31,0,1,\"Uruchurtu, Don. Manuel E\",male,40,0,0,PC 17601,27.7208,,C\n32,1,1,\"Spencer, Mrs. William Augustus (Marie Eugenie)\",female,,1,0,PC 17569,146.5208,B78,C\n33,1,3,\"Glynn, Miss. Mary Agatha\",female,,0,0,335677,7.75,,Q\n34,0,2,\"Wheadon, Mr. Edward H\",male,66,0,0,C.A. 24579,10.5,,S\n35,0,1,\"Meyer, Mr. Edgar Joseph\",male,28,1,0,PC 17604,82.1708,,C\n36,0,1,\"Holverson, Mr. Alexander Oskar\",male,42,1,0,113789,52,,S\n37,1,3,\"Mamee, Mr. Hanna\",male,,0,0,2677,7.2292,,C\n38,0,3,\"Cann, Mr. Ernest Charles\",male,21,0,0,A./5. 2152,8.05,,S\n39,0,3,\"Vander Planke, Miss. Augusta Maria\",female,18,2,0,345764,18,,S\n40,1,3,\"Nicola-Yarred, Miss. Jamila\",female,14,1,0,2651,11.2417,,C\n41,0,3,\"Ahlin, Mrs. Johan (Johanna Persdotter Larsson)\",female,40,1,0,7546,9.475,,S\n42,0,2,\"Turpin, Mrs. William John Robert (Dorothy Ann Wonnacott)\",female,27,1,0,11668,21,,S\n43,0,3,\"Kraeff, Mr. Theodor\",male,,0,0,349253,7.8958,,C\n44,1,2,\"Laroche, Miss. Simonne Marie Anne Andree\",female,3,1,2,SC/Paris 2123,41.5792,,C\n45,1,3,\"Devaney, Miss. Margaret Delia\",female,19,0,0,330958,7.8792,,Q\n46,0,3,\"Rogers, Mr. William John\",male,,0,0,S.C./A.4. 23567,8.05,,S\n47,0,3,\"Lennon, Mr. Denis\",male,,1,0,370371,15.5,,Q\n48,1,3,\"O'Driscoll, Miss. Bridget\",female,,0,0,14311,7.75,,Q\n49,0,3,\"Samaan, Mr. Youssef\",male,,2,0,2662,21.6792,,C\n50,0,3,\"Arnold-Franchi, Mrs. Josef (Josefine Franchi)\",female,18,1,0,349237,17.8,,S\n51,0,3,\"Panula, Master. Juha Niilo\",male,7,4,1,3101295,39.6875,,S\n52,0,3,\"Nosworthy, Mr. Richard Cater\",male,21,0,0,A/4. 39886,7.8,,S\n53,1,1,\"Harper, Mrs. Henry Sleeper (Myna Haxtun)\",female,49,1,0,PC 17572,76.7292,D33,C\n54,1,2,\"Faunthorpe, Mrs. Lizzie (Elizabeth Anne Wilkinson)\",female,29,1,0,2926,26,,S\n55,0,1,\"Ostby, Mr. Engelhart Cornelius\",male,65,0,1,113509,61.9792,B30,C\n56,1,1,\"Woolner, Mr. Hugh\",male,,0,0,19947,35.5,C52,S\n57,1,2,\"Rugg, Miss. Emily\",female,21,0,0,C.A. 31026,10.5,,S\n58,0,3,\"Novel, Mr. Mansouer\",male,28.5,0,0,2697,7.2292,,C\n59,1,2,\"West, Miss. Constance Mirium\",female,5,1,2,C.A. 34651,27.75,,S\n60,0,3,\"Goodwin, Master. William Frederick\",male,11,5,2,CA 2144,46.9,,S\n61,0,3,\"Sirayanian, Mr. Orsen\",male,22,0,0,2669,7.2292,,C\n62,1,1,\"Icard, Miss. Amelie\",female,38,0,0,113572,80,B28,\n63,0,1,\"Harris, Mr. Henry Birkhardt\",male,45,1,0,36973,83.475,C83,S\n64,0,3,\"Skoog, Master. Harald\",male,4,3,2,347088,27.9,,S\n65,0,1,\"Stewart, Mr. Albert A\",male,,0,0,PC 17605,27.7208,,C\n66,1,3,\"Moubarek, Master. Gerios\",male,,1,1,2661,15.2458,,C\n67,1,2,\"Nye, Mrs. (Elizabeth Ramell)\",female,29,0,0,C.A. 29395,10.5,F33,S\n68,0,3,\"Crease, Mr. Ernest James\",male,19,0,0,S.P. 3464,8.1583,,S\n69,1,3,\"Andersson, Miss. Erna Alexandra\",female,17,4,2,3101281,7.925,,S\n70,0,3,\"Kink, Mr. Vincenz\",male,26,2,0,315151,8.6625,,S\n71,0,2,\"Jenkin, Mr. Stephen Curnow\",male,32,0,0,C.A. 33111,10.5,,S\n72,0,3,\"Goodwin, Miss. Lillian Amy\",female,16,5,2,CA 2144,46.9,,S\n73,0,2,\"Hood, Mr. Ambrose Jr\",male,21,0,0,S.O.C. 14879,73.5,,S\n74,0,3,\"Chronopoulos, Mr. Apostolos\",male,26,1,0,2680,14.4542,,C\n75,1,3,\"Bing, Mr. Lee\",male,32,0,0,1601,56.4958,,S\n76,0,3,\"Moen, Mr. Sigurd Hansen\",male,25,0,0,348123,7.65,F G73,S\n77,0,3,\"Staneff, Mr. Ivan\",male,,0,0,349208,7.8958,,S\n78,0,3,\"Moutal, Mr. Rahamin Haim\",male,,0,0,374746,8.05,,S\n79,1,2,\"Caldwell, Master. Alden Gates\",male,0.83,0,2,248738,29,,S\n80,1,3,\"Dowdell, Miss. Elizabeth\",female,30,0,0,364516,12.475,,S\n81,0,3,\"Waelens, Mr. Achille\",male,22,0,0,345767,9,,S\n82,1,3,\"Sheerlinck, Mr. Jan Baptist\",male,29,0,0,345779,9.5,,S\n83,1,3,\"McDermott, Miss. Brigdet Delia\",female,,0,0,330932,7.7875,,Q\n84,0,1,\"Carrau, Mr. Francisco M\",male,28,0,0,113059,47.1,,S\n85,1,2,\"Ilett, Miss. Bertha\",female,17,0,0,SO/C 14885,10.5,,S\n86,1,3,\"Backstrom, Mrs. Karl Alfred (Maria Mathilda Gustafsson)\",female,33,3,0,3101278,15.85,,S\n87,0,3,\"Ford, Mr. William Neal\",male,16,1,3,W./C. 6608,34.375,,S\n88,0,3,\"Slocovski, Mr. Selman Francis\",male,,0,0,SOTON/OQ 392086,8.05,,S\n89,1,1,\"Fortune, Miss. Mabel Helen\",female,23,3,2,19950,263,C23 C25 C27,S\n90,0,3,\"Celotti, Mr. Francesco\",male,24,0,0,343275,8.05,,S\n91,0,3,\"Christmann, Mr. Emil\",male,29,0,0,343276,8.05,,S\n92,0,3,\"Andreasson, Mr. Paul Edvin\",male,20,0,0,347466,7.8542,,S\n93,0,1,\"Chaffee, Mr. Herbert Fuller\",male,46,1,0,W.E.P. 5734,61.175,E31,S\n94,0,3,\"Dean, Mr. Bertram Frank\",male,26,1,2,C.A. 2315,20.575,,S\n95,0,3,\"Coxon, Mr. Daniel\",male,59,0,0,364500,7.25,,S\n96,0,3,\"Shorney, Mr. Charles Joseph\",male,,0,0,374910,8.05,,S\n97,0,1,\"Goldschmidt, Mr. George B\",male,71,0,0,PC 17754,34.6542,A5,C\n98,1,1,\"Greenfield, Mr. William Bertram\",male,23,0,1,PC 17759,63.3583,D10 D12,C\n99,1,2,\"Doling, Mrs. John T (Ada Julia Bone)\",female,34,0,1,231919,23,,S\n100,0,2,\"Kantor, Mr. Sinai\",male,34,1,0,244367,26,,S\n101,0,3,\"Petranec, Miss. Matilda\",female,28,0,0,349245,7.8958,,S\n102,0,3,\"Petroff, Mr. Pastcho (\"\"Pentcho\"\")\",male,,0,0,349215,7.8958,,S\n103,0,1,\"White, Mr. Richard Frasar\",male,21,0,1,35281,77.2875,D26,S\n104,0,3,\"Johansson, Mr. Gustaf Joel\",male,33,0,0,7540,8.6542,,S\n105,0,3,\"Gustafsson, Mr. Anders Vilhelm\",male,37,2,0,3101276,7.925,,S\n106,0,3,\"Mionoff, Mr. Stoytcho\",male,28,0,0,349207,7.8958,,S\n107,1,3,\"Salkjelsvik, Miss. Anna Kristine\",female,21,0,0,343120,7.65,,S\n108,1,3,\"Moss, Mr. Albert Johan\",male,,0,0,312991,7.775,,S\n109,0,3,\"Rekic, Mr. Tido\",male,38,0,0,349249,7.8958,,S\n110,1,3,\"Moran, Miss. Bertha\",female,,1,0,371110,24.15,,Q\n111,0,1,\"Porter, Mr. Walter Chamberlain\",male,47,0,0,110465,52,C110,S\n112,0,3,\"Zabour, Miss. Hileni\",female,14.5,1,0,2665,14.4542,,C\n113,0,3,\"Barton, Mr. David John\",male,22,0,0,324669,8.05,,S\n114,0,3,\"Jussila, Miss. Katriina\",female,20,1,0,4136,9.825,,S\n115,0,3,\"Attalah, Miss. Malake\",female,17,0,0,2627,14.4583,,C\n116,0,3,\"Pekoniemi, Mr. Edvard\",male,21,0,0,STON/O 2. 3101294,7.925,,S\n117,0,3,\"Connors, Mr. Patrick\",male,70.5,0,0,370369,7.75,,Q\n118,0,2,\"Turpin, Mr. William John Robert\",male,29,1,0,11668,21,,S\n119,0,1,\"Baxter, Mr. Quigg Edmond\",male,24,0,1,PC 17558,247.5208,B58 B60,C\n120,0,3,\"Andersson, Miss. Ellis Anna Maria\",female,2,4,2,347082,31.275,,S\n121,0,2,\"Hickman, Mr. Stanley George\",male,21,2,0,S.O.C. 14879,73.5,,S\n122,0,3,\"Moore, Mr. Leonard Charles\",male,,0,0,A4. 54510,8.05,,S\n123,0,2,\"Nasser, Mr. Nicholas\",male,32.5,1,0,237736,30.0708,,C\n124,1,2,\"Webber, Miss. Susan\",female,32.5,0,0,27267,13,E101,S\n125,0,1,\"White, Mr. Percival Wayland\",male,54,0,1,35281,77.2875,D26,S\n126,1,3,\"Nicola-Yarred, Master. Elias\",male,12,1,0,2651,11.2417,,C\n127,0,3,\"McMahon, Mr. Martin\",male,,0,0,370372,7.75,,Q\n128,1,3,\"Madsen, Mr. Fridtjof Arne\",male,24,0,0,C 17369,7.1417,,S\n129,1,3,\"Peter, Miss. Anna\",female,,1,1,2668,22.3583,F E69,C\n130,0,3,\"Ekstrom, Mr. Johan\",male,45,0,0,347061,6.975,,S\n131,0,3,\"Drazenoic, Mr. Jozef\",male,33,0,0,349241,7.8958,,C\n132,0,3,\"Coelho, Mr. Domingos Fernandeo\",male,20,0,0,SOTON/O.Q. 3101307,7.05,,S\n133,0,3,\"Robins, Mrs. Alexander A (Grace Charity Laury)\",female,47,1,0,A/5. 3337,14.5,,S\n134,1,2,\"Weisz, Mrs. Leopold (Mathilde Francoise Pede)\",female,29,1,0,228414,26,,S\n135,0,2,\"Sobey, Mr. Samuel James Hayden\",male,25,0,0,C.A. 29178,13,,S\n136,0,2,\"Richard, Mr. Emile\",male,23,0,0,SC/PARIS 2133,15.0458,,C\n137,1,1,\"Newsom, Miss. Helen Monypeny\",female,19,0,2,11752,26.2833,D47,S\n138,0,1,\"Futrelle, Mr. Jacques Heath\",male,37,1,0,113803,53.1,C123,S\n139,0,3,\"Osen, Mr. Olaf Elon\",male,16,0,0,7534,9.2167,,S\n140,0,1,\"Giglio, Mr. Victor\",male,24,0,0,PC 17593,79.2,B86,C\n141,0,3,\"Boulos, Mrs. Joseph (Sultana)\",female,,0,2,2678,15.2458,,C\n142,1,3,\"Nysten, Miss. Anna Sofia\",female,22,0,0,347081,7.75,,S\n143,1,3,\"Hakkarainen, Mrs. Pekka Pietari (Elin Matilda Dolck)\",female,24,1,0,STON/O2. 3101279,15.85,,S\n144,0,3,\"Burke, Mr. Jeremiah\",male,19,0,0,365222,6.75,,Q\n145,0,2,\"Andrew, Mr. Edgardo Samuel\",male,18,0,0,231945,11.5,,S\n146,0,2,\"Nicholls, Mr. Joseph Charles\",male,19,1,1,C.A. 33112,36.75,,S\n147,1,3,\"Andersson, Mr. August Edvard (\"\"Wennerstrom\"\")\",male,27,0,0,350043,7.7958,,S\n148,0,3,\"Ford, Miss. Robina Maggie \"\"Ruby\"\"\",female,9,2,2,W./C. 6608,34.375,,S\n149,0,2,\"Navratil, Mr. Michel (\"\"Louis M Hoffman\"\")\",male,36.5,0,2,230080,26,F2,S\n150,0,2,\"Byles, Rev. Thomas Roussel Davids\",male,42,0,0,244310,13,,S\n151,0,2,\"Bateman, Rev. Robert James\",male,51,0,0,S.O.P. 1166,12.525,,S\n152,1,1,\"Pears, Mrs. Thomas (Edith Wearne)\",female,22,1,0,113776,66.6,C2,S\n153,0,3,\"Meo, Mr. Alfonzo\",male,55.5,0,0,A.5. 11206,8.05,,S\n154,0,3,\"van Billiard, Mr. Austin Blyler\",male,40.5,0,2,A/5. 851,14.5,,S\n155,0,3,\"Olsen, Mr. Ole Martin\",male,,0,0,Fa 265302,7.3125,,S\n156,0,1,\"Williams, Mr. Charles Duane\",male,51,0,1,PC 17597,61.3792,,C\n157,1,3,\"Gilnagh, Miss. Katherine \"\"Katie\"\"\",female,16,0,0,35851,7.7333,,Q\n158,0,3,\"Corn, Mr. Harry\",male,30,0,0,SOTON/OQ 392090,8.05,,S\n159,0,3,\"Smiljanic, Mr. Mile\",male,,0,0,315037,8.6625,,S\n160,0,3,\"Sage, Master. Thomas Henry\",male,,8,2,CA. 2343,69.55,,S\n161,0,3,\"Cribb, Mr. John Hatfield\",male,44,0,1,371362,16.1,,S\n162,1,2,\"Watt, Mrs. James (Elizabeth \"\"Bessie\"\" Inglis Milne)\",female,40,0,0,C.A. 33595,15.75,,S\n163,0,3,\"Bengtsson, Mr. John Viktor\",male,26,0,0,347068,7.775,,S\n164,0,3,\"Calic, Mr. Jovo\",male,17,0,0,315093,8.6625,,S\n165,0,3,\"Panula, Master. Eino Viljami\",male,1,4,1,3101295,39.6875,,S\n166,1,3,\"Goldsmith, Master. Frank John William \"\"Frankie\"\"\",male,9,0,2,363291,20.525,,S\n167,1,1,\"Chibnall, Mrs. (Edith Martha Bowerman)\",female,,0,1,113505,55,E33,S\n168,0,3,\"Skoog, Mrs. William (Anna Bernhardina Karlsson)\",female,45,1,4,347088,27.9,,S\n169,0,1,\"Baumann, Mr. John D\",male,,0,0,PC 17318,25.925,,S\n170,0,3,\"Ling, Mr. Lee\",male,28,0,0,1601,56.4958,,S\n171,0,1,\"Van der hoef, Mr. Wyckoff\",male,61,0,0,111240,33.5,B19,S\n172,0,3,\"Rice, Master. Arthur\",male,4,4,1,382652,29.125,,Q\n173,1,3,\"Johnson, Miss. Eleanor Ileen\",female,1,1,1,347742,11.1333,,S\n174,0,3,\"Sivola, Mr. Antti Wilhelm\",male,21,0,0,STON/O 2. 3101280,7.925,,S\n175,0,1,\"Smith, Mr. James Clinch\",male,56,0,0,17764,30.6958,A7,C\n176,0,3,\"Klasen, Mr. Klas Albin\",male,18,1,1,350404,7.8542,,S\n177,0,3,\"Lefebre, Master. Henry Forbes\",male,,3,1,4133,25.4667,,S\n178,0,1,\"Isham, Miss. Ann Elizabeth\",female,50,0,0,PC 17595,28.7125,C49,C\n179,0,2,\"Hale, Mr. Reginald\",male,30,0,0,250653,13,,S\n180,0,3,\"Leonard, Mr. Lionel\",male,36,0,0,LINE,0,,S\n181,0,3,\"Sage, Miss. Constance Gladys\",female,,8,2,CA. 2343,69.55,,S\n182,0,2,\"Pernot, Mr. Rene\",male,,0,0,SC/PARIS 2131,15.05,,C\n183,0,3,\"Asplund, Master. Clarence Gustaf Hugo\",male,9,4,2,347077,31.3875,,S\n184,1,2,\"Becker, Master. Richard F\",male,1,2,1,230136,39,F4,S\n185,1,3,\"Kink-Heilmann, Miss. Luise Gretchen\",female,4,0,2,315153,22.025,,S\n186,0,1,\"Rood, Mr. Hugh Roscoe\",male,,0,0,113767,50,A32,S\n187,1,3,\"O'Brien, Mrs. Thomas (Johanna \"\"Hannah\"\" Godfrey)\",female,,1,0,370365,15.5,,Q\n188,1,1,\"Romaine, Mr. Charles Hallace (\"\"Mr C Rolmane\"\")\",male,45,0,0,111428,26.55,,S\n189,0,3,\"Bourke, Mr. John\",male,40,1,1,364849,15.5,,Q\n190,0,3,\"Turcin, Mr. Stjepan\",male,36,0,0,349247,7.8958,,S\n191,1,2,\"Pinsky, Mrs. (Rosa)\",female,32,0,0,234604,13,,S\n192,0,2,\"Carbines, Mr. William\",male,19,0,0,28424,13,,S\n193,1,3,\"Andersen-Jensen, Miss. Carla Christine Nielsine\",female,19,1,0,350046,7.8542,,S\n194,1,2,\"Navratil, Master. Michel M\",male,3,1,1,230080,26,F2,S\n195,1,1,\"Brown, Mrs. James Joseph (Margaret Tobin)\",female,44,0,0,PC 17610,27.7208,B4,C\n196,1,1,\"Lurette, Miss. Elise\",female,58,0,0,PC 17569,146.5208,B80,C\n197,0,3,\"Mernagh, Mr. Robert\",male,,0,0,368703,7.75,,Q\n198,0,3,\"Olsen, Mr. Karl Siegwart Andreas\",male,42,0,1,4579,8.4042,,S\n199,1,3,\"Madigan, Miss. Margaret \"\"Maggie\"\"\",female,,0,0,370370,7.75,,Q\n200,0,2,\"Yrois, Miss. Henriette (\"\"Mrs Harbeck\"\")\",female,24,0,0,248747,13,,S\n201,0,3,\"Vande Walle, Mr. Nestor Cyriel\",male,28,0,0,345770,9.5,,S\n202,0,3,\"Sage, Mr. Frederick\",male,,8,2,CA. 2343,69.55,,S\n203,0,3,\"Johanson, Mr. Jakob Alfred\",male,34,0,0,3101264,6.4958,,S\n204,0,3,\"Youseff, Mr. Gerious\",male,45.5,0,0,2628,7.225,,C\n205,1,3,\"Cohen, Mr. Gurshon \"\"Gus\"\"\",male,18,0,0,A/5 3540,8.05,,S\n206,0,3,\"Strom, Miss. Telma Matilda\",female,2,0,1,347054,10.4625,G6,S\n207,0,3,\"Backstrom, Mr. Karl Alfred\",male,32,1,0,3101278,15.85,,S\n208,1,3,\"Albimona, Mr. Nassef Cassem\",male,26,0,0,2699,18.7875,,C\n209,1,3,\"Carr, Miss. Helen \"\"Ellen\"\"\",female,16,0,0,367231,7.75,,Q\n210,1,1,\"Blank, Mr. Henry\",male,40,0,0,112277,31,A31,C\n211,0,3,\"Ali, Mr. Ahmed\",male,24,0,0,SOTON/O.Q. 3101311,7.05,,S\n212,1,2,\"Cameron, Miss. Clear Annie\",female,35,0,0,F.C.C. 13528,21,,S\n213,0,3,\"Perkin, Mr. John Henry\",male,22,0,0,A/5 21174,7.25,,S\n214,0,2,\"Givard, Mr. Hans Kristensen\",male,30,0,0,250646,13,,S\n215,0,3,\"Kiernan, Mr. Philip\",male,,1,0,367229,7.75,,Q\n216,1,1,\"Newell, Miss. Madeleine\",female,31,1,0,35273,113.275,D36,C\n217,1,3,\"Honkanen, Miss. Eliina\",female,27,0,0,STON/O2. 3101283,7.925,,S\n218,0,2,\"Jacobsohn, Mr. Sidney Samuel\",male,42,1,0,243847,27,,S\n219,1,1,\"Bazzani, Miss. Albina\",female,32,0,0,11813,76.2917,D15,C\n220,0,2,\"Harris, Mr. Walter\",male,30,0,0,W/C 14208,10.5,,S\n221,1,3,\"Sunderland, Mr. Victor Francis\",male,16,0,0,SOTON/OQ 392089,8.05,,S\n222,0,2,\"Bracken, Mr. James H\",male,27,0,0,220367,13,,S\n223,0,3,\"Green, Mr. George Henry\",male,51,0,0,21440,8.05,,S\n224,0,3,\"Nenkoff, Mr. Christo\",male,,0,0,349234,7.8958,,S\n225,1,1,\"Hoyt, Mr. Frederick Maxfield\",male,38,1,0,19943,90,C93,S\n226,0,3,\"Berglund, Mr. Karl Ivar Sven\",male,22,0,0,PP 4348,9.35,,S\n227,1,2,\"Mellors, Mr. William John\",male,19,0,0,SW/PP 751,10.5,,S\n228,0,3,\"Lovell, Mr. John Hall (\"\"Henry\"\")\",male,20.5,0,0,A/5 21173,7.25,,S\n229,0,2,\"Fahlstrom, Mr. Arne Jonas\",male,18,0,0,236171,13,,S\n230,0,3,\"Lefebre, Miss. Mathilde\",female,,3,1,4133,25.4667,,S\n231,1,1,\"Harris, Mrs. Henry Birkhardt (Irene Wallach)\",female,35,1,0,36973,83.475,C83,S\n232,0,3,\"Larsson, Mr. Bengt Edvin\",male,29,0,0,347067,7.775,,S\n233,0,2,\"Sjostedt, Mr. Ernst Adolf\",male,59,0,0,237442,13.5,,S\n234,1,3,\"Asplund, Miss. Lillian Gertrud\",female,5,4,2,347077,31.3875,,S\n235,0,2,\"Leyson, Mr. Robert William Norman\",male,24,0,0,C.A. 29566,10.5,,S\n236,0,3,\"Harknett, Miss. Alice Phoebe\",female,,0,0,W./C. 6609,7.55,,S\n237,0,2,\"Hold, Mr. Stephen\",male,44,1,0,26707,26,,S\n238,1,2,\"Collyer, Miss. Marjorie \"\"Lottie\"\"\",female,8,0,2,C.A. 31921,26.25,,S\n239,0,2,\"Pengelly, Mr. Frederick William\",male,19,0,0,28665,10.5,,S\n240,0,2,\"Hunt, Mr. George Henry\",male,33,0,0,SCO/W 1585,12.275,,S\n241,0,3,\"Zabour, Miss. Thamine\",female,,1,0,2665,14.4542,,C\n242,1,3,\"Murphy, Miss. Katherine \"\"Kate\"\"\",female,,1,0,367230,15.5,,Q\n243,0,2,\"Coleridge, Mr. Reginald Charles\",male,29,0,0,W./C. 14263,10.5,,S\n244,0,3,\"Maenpaa, Mr. Matti Alexanteri\",male,22,0,0,STON/O 2. 3101275,7.125,,S\n245,0,3,\"Attalah, Mr. Sleiman\",male,30,0,0,2694,7.225,,C\n246,0,1,\"Minahan, Dr. William Edward\",male,44,2,0,19928,90,C78,Q\n247,0,3,\"Lindahl, Miss. Agda Thorilda Viktoria\",female,25,0,0,347071,7.775,,S\n248,1,2,\"Hamalainen, Mrs. William (Anna)\",female,24,0,2,250649,14.5,,S\n249,1,1,\"Beckwith, Mr. Richard Leonard\",male,37,1,1,11751,52.5542,D35,S\n250,0,2,\"Carter, Rev. Ernest Courtenay\",male,54,1,0,244252,26,,S\n251,0,3,\"Reed, Mr. James George\",male,,0,0,362316,7.25,,S\n252,0,3,\"Strom, Mrs. Wilhelm (Elna Matilda Persson)\",female,29,1,1,347054,10.4625,G6,S\n253,0,1,\"Stead, Mr. William Thomas\",male,62,0,0,113514,26.55,C87,S\n254,0,3,\"Lobb, Mr. William Arthur\",male,30,1,0,A/5. 3336,16.1,,S\n255,0,3,\"Rosblom, Mrs. Viktor (Helena Wilhelmina)\",female,41,0,2,370129,20.2125,,S\n256,1,3,\"Touma, Mrs. Darwis (Hanne Youssef Razi)\",female,29,0,2,2650,15.2458,,C\n257,1,1,\"Thorne, Mrs. Gertrude Maybelle\",female,,0,0,PC 17585,79.2,,C\n258,1,1,\"Cherry, Miss. Gladys\",female,30,0,0,110152,86.5,B77,S\n259,1,1,\"Ward, Miss. Anna\",female,35,0,0,PC 17755,512.3292,,C\n260,1,2,\"Parrish, Mrs. (Lutie Davis)\",female,50,0,1,230433,26,,S\n261,0,3,\"Smith, Mr. Thomas\",male,,0,0,384461,7.75,,Q\n262,1,3,\"Asplund, Master. Edvin Rojj Felix\",male,3,4,2,347077,31.3875,,S\n263,0,1,\"Taussig, Mr. Emil\",male,52,1,1,110413,79.65,E67,S\n264,0,1,\"Harrison, Mr. William\",male,40,0,0,112059,0,B94,S\n265,0,3,\"Henry, Miss. Delia\",female,,0,0,382649,7.75,,Q\n266,0,2,\"Reeves, Mr. David\",male,36,0,0,C.A. 17248,10.5,,S\n267,0,3,\"Panula, Mr. Ernesti Arvid\",male,16,4,1,3101295,39.6875,,S\n268,1,3,\"Persson, Mr. Ernst Ulrik\",male,25,1,0,347083,7.775,,S\n269,1,1,\"Graham, Mrs. William Thompson (Edith Junkins)\",female,58,0,1,PC 17582,153.4625,C125,S\n270,1,1,\"Bissette, Miss. Amelia\",female,35,0,0,PC 17760,135.6333,C99,S\n271,0,1,\"Cairns, Mr. Alexander\",male,,0,0,113798,31,,S\n272,1,3,\"Tornquist, Mr. William Henry\",male,25,0,0,LINE,0,,S\n273,1,2,\"Mellinger, Mrs. (Elizabeth Anne Maidment)\",female,41,0,1,250644,19.5,,S\n274,0,1,\"Natsch, Mr. Charles H\",male,37,0,1,PC 17596,29.7,C118,C\n275,1,3,\"Healy, Miss. Hanora \"\"Nora\"\"\",female,,0,0,370375,7.75,,Q\n276,1,1,\"Andrews, Miss. Kornelia Theodosia\",female,63,1,0,13502,77.9583,D7,S\n277,0,3,\"Lindblom, Miss. Augusta Charlotta\",female,45,0,0,347073,7.75,,S\n278,0,2,\"Parkes, Mr. Francis \"\"Frank\"\"\",male,,0,0,239853,0,,S\n279,0,3,\"Rice, Master. Eric\",male,7,4,1,382652,29.125,,Q\n280,1,3,\"Abbott, Mrs. Stanton (Rosa Hunt)\",female,35,1,1,C.A. 2673,20.25,,S\n281,0,3,\"Duane, Mr. Frank\",male,65,0,0,336439,7.75,,Q\n282,0,3,\"Olsson, Mr. Nils Johan Goransson\",male,28,0,0,347464,7.8542,,S\n283,0,3,\"de Pelsmaeker, Mr. Alfons\",male,16,0,0,345778,9.5,,S\n284,1,3,\"Dorking, Mr. Edward Arthur\",male,19,0,0,A/5. 10482,8.05,,S\n285,0,1,\"Smith, Mr. Richard William\",male,,0,0,113056,26,A19,S\n286,0,3,\"Stankovic, Mr. Ivan\",male,33,0,0,349239,8.6625,,C\n287,1,3,\"de Mulder, Mr. Theodore\",male,30,0,0,345774,9.5,,S\n288,0,3,\"Naidenoff, Mr. Penko\",male,22,0,0,349206,7.8958,,S\n289,1,2,\"Hosono, Mr. Masabumi\",male,42,0,0,237798,13,,S\n290,1,3,\"Connolly, Miss. Kate\",female,22,0,0,370373,7.75,,Q\n291,1,1,\"Barber, Miss. Ellen \"\"Nellie\"\"\",female,26,0,0,19877,78.85,,S\n292,1,1,\"Bishop, Mrs. Dickinson H (Helen Walton)\",female,19,1,0,11967,91.0792,B49,C\n293,0,2,\"Levy, Mr. Rene Jacques\",male,36,0,0,SC/Paris 2163,12.875,D,C\n294,0,3,\"Haas, Miss. Aloisia\",female,24,0,0,349236,8.85,,S\n295,0,3,\"Mineff, Mr. Ivan\",male,24,0,0,349233,7.8958,,S\n296,0,1,\"Lewy, Mr. Ervin G\",male,,0,0,PC 17612,27.7208,,C\n297,0,3,\"Hanna, Mr. Mansour\",male,23.5,0,0,2693,7.2292,,C\n298,0,1,\"Allison, Miss. Helen Loraine\",female,2,1,2,113781,151.55,C22 C26,S\n299,1,1,\"Saalfeld, Mr. Adolphe\",male,,0,0,19988,30.5,C106,S\n300,1,1,\"Baxter, Mrs. James (Helene DeLaudeniere Chaput)\",female,50,0,1,PC 17558,247.5208,B58 B60,C\n301,1,3,\"Kelly, Miss. Anna Katherine \"\"Annie Kate\"\"\",female,,0,0,9234,7.75,,Q\n302,1,3,\"McCoy, Mr. Bernard\",male,,2,0,367226,23.25,,Q\n303,0,3,\"Johnson, Mr. William Cahoone Jr\",male,19,0,0,LINE,0,,S\n304,1,2,\"Keane, Miss. Nora A\",female,,0,0,226593,12.35,E101,Q\n305,0,3,\"Williams, Mr. Howard Hugh \"\"Harry\"\"\",male,,0,0,A/5 2466,8.05,,S\n306,1,1,\"Allison, Master. Hudson Trevor\",male,0.92,1,2,113781,151.55,C22 C26,S\n307,1,1,\"Fleming, Miss. Margaret\",female,,0,0,17421,110.8833,,C\n308,1,1,\"Penasco y Castellana, Mrs. Victor de Satode (Maria Josefa Perez de Soto y Vallejo)\",female,17,1,0,PC 17758,108.9,C65,C\n309,0,2,\"Abelson, Mr. Samuel\",male,30,1,0,P/PP 3381,24,,C\n310,1,1,\"Francatelli, Miss. Laura Mabel\",female,30,0,0,PC 17485,56.9292,E36,C\n311,1,1,\"Hays, Miss. Margaret Bechstein\",female,24,0,0,11767,83.1583,C54,C\n312,1,1,\"Ryerson, Miss. Emily Borie\",female,18,2,2,PC 17608,262.375,B57 B59 B63 B66,C\n313,0,2,\"Lahtinen, Mrs. William (Anna Sylfven)\",female,26,1,1,250651,26,,S\n314,0,3,\"Hendekovic, Mr. Ignjac\",male,28,0,0,349243,7.8958,,S\n315,0,2,\"Hart, Mr. Benjamin\",male,43,1,1,F.C.C. 13529,26.25,,S\n316,1,3,\"Nilsson, Miss. Helmina Josefina\",female,26,0,0,347470,7.8542,,S\n317,1,2,\"Kantor, Mrs. Sinai (Miriam Sternin)\",female,24,1,0,244367,26,,S\n318,0,2,\"Moraweck, Dr. Ernest\",male,54,0,0,29011,14,,S\n319,1,1,\"Wick, Miss. Mary Natalie\",female,31,0,2,36928,164.8667,C7,S\n320,1,1,\"Spedden, Mrs. Frederic Oakley (Margaretta Corning Stone)\",female,40,1,1,16966,134.5,E34,C\n321,0,3,\"Dennis, Mr. Samuel\",male,22,0,0,A/5 21172,7.25,,S\n322,0,3,\"Danoff, Mr. Yoto\",male,27,0,0,349219,7.8958,,S\n323,1,2,\"Slayter, Miss. Hilda Mary\",female,30,0,0,234818,12.35,,Q\n324,1,2,\"Caldwell, Mrs. Albert Francis (Sylvia Mae Harbaugh)\",female,22,1,1,248738,29,,S\n325,0,3,\"Sage, Mr. George John Jr\",male,,8,2,CA. 2343,69.55,,S\n326,1,1,\"Young, Miss. Marie Grice\",female,36,0,0,PC 17760,135.6333,C32,C\n327,0,3,\"Nysveen, Mr. Johan Hansen\",male,61,0,0,345364,6.2375,,S\n328,1,2,\"Ball, Mrs. (Ada E Hall)\",female,36,0,0,28551,13,D,S\n329,1,3,\"Goldsmith, Mrs. Frank John (Emily Alice Brown)\",female,31,1,1,363291,20.525,,S\n330,1,1,\"Hippach, Miss. Jean Gertrude\",female,16,0,1,111361,57.9792,B18,C\n331,1,3,\"McCoy, Miss. Agnes\",female,,2,0,367226,23.25,,Q\n332,0,1,\"Partner, Mr. Austen\",male,45.5,0,0,113043,28.5,C124,S\n333,0,1,\"Graham, Mr. George Edward\",male,38,0,1,PC 17582,153.4625,C91,S\n334,0,3,\"Vander Planke, Mr. Leo Edmondus\",male,16,2,0,345764,18,,S\n335,1,1,\"Frauenthal, Mrs. Henry William (Clara Heinsheimer)\",female,,1,0,PC 17611,133.65,,S\n336,0,3,\"Denkoff, Mr. Mitto\",male,,0,0,349225,7.8958,,S\n337,0,1,\"Pears, Mr. Thomas Clinton\",male,29,1,0,113776,66.6,C2,S\n338,1,1,\"Burns, Miss. Elizabeth Margaret\",female,41,0,0,16966,134.5,E40,C\n339,1,3,\"Dahl, Mr. Karl Edwart\",male,45,0,0,7598,8.05,,S\n340,0,1,\"Blackwell, Mr. Stephen Weart\",male,45,0,0,113784,35.5,T,S\n341,1,2,\"Navratil, Master. Edmond Roger\",male,2,1,1,230080,26,F2,S\n342,1,1,\"Fortune, Miss. Alice Elizabeth\",female,24,3,2,19950,263,C23 C25 C27,S\n343,0,2,\"Collander, Mr. Erik Gustaf\",male,28,0,0,248740,13,,S\n344,0,2,\"Sedgwick, Mr. Charles Frederick Waddington\",male,25,0,0,244361,13,,S\n345,0,2,\"Fox, Mr. Stanley Hubert\",male,36,0,0,229236,13,,S\n346,1,2,\"Brown, Miss. Amelia \"\"Mildred\"\"\",female,24,0,0,248733,13,F33,S\n347,1,2,\"Smith, Miss. Marion Elsie\",female,40,0,0,31418,13,,S\n348,1,3,\"Davison, Mrs. Thomas Henry (Mary E Finck)\",female,,1,0,386525,16.1,,S\n349,1,3,\"Coutts, Master. William Loch \"\"William\"\"\",male,3,1,1,C.A. 37671,15.9,,S\n350,0,3,\"Dimic, Mr. Jovan\",male,42,0,0,315088,8.6625,,S\n351,0,3,\"Odahl, Mr. Nils Martin\",male,23,0,0,7267,9.225,,S\n352,0,1,\"Williams-Lambert, Mr. Fletcher Fellows\",male,,0,0,113510,35,C128,S\n353,0,3,\"Elias, Mr. Tannous\",male,15,1,1,2695,7.2292,,C\n354,0,3,\"Arnold-Franchi, Mr. Josef\",male,25,1,0,349237,17.8,,S\n355,0,3,\"Yousif, Mr. Wazli\",male,,0,0,2647,7.225,,C\n356,0,3,\"Vanden Steen, Mr. Leo Peter\",male,28,0,0,345783,9.5,,S\n357,1,1,\"Bowerman, Miss. Elsie Edith\",female,22,0,1,113505,55,E33,S\n358,0,2,\"Funk, Miss. Annie Clemmer\",female,38,0,0,237671,13,,S\n359,1,3,\"McGovern, Miss. Mary\",female,,0,0,330931,7.8792,,Q\n360,1,3,\"Mockler, Miss. Helen Mary \"\"Ellie\"\"\",female,,0,0,330980,7.8792,,Q\n361,0,3,\"Skoog, Mr. Wilhelm\",male,40,1,4,347088,27.9,,S\n362,0,2,\"del Carlo, Mr. Sebastiano\",male,29,1,0,SC/PARIS 2167,27.7208,,C\n363,0,3,\"Barbara, Mrs. (Catherine David)\",female,45,0,1,2691,14.4542,,C\n364,0,3,\"Asim, Mr. Adola\",male,35,0,0,SOTON/O.Q. 3101310,7.05,,S\n365,0,3,\"O'Brien, Mr. Thomas\",male,,1,0,370365,15.5,,Q\n366,0,3,\"Adahl, Mr. Mauritz Nils Martin\",male,30,0,0,C 7076,7.25,,S\n367,1,1,\"Warren, Mrs. Frank Manley (Anna Sophia Atkinson)\",female,60,1,0,110813,75.25,D37,C\n368,1,3,\"Moussa, Mrs. (Mantoura Boulos)\",female,,0,0,2626,7.2292,,C\n369,1,3,\"Jermyn, Miss. Annie\",female,,0,0,14313,7.75,,Q\n370,1,1,\"Aubart, Mme. Leontine Pauline\",female,24,0,0,PC 17477,69.3,B35,C\n371,1,1,\"Harder, Mr. George Achilles\",male,25,1,0,11765,55.4417,E50,C\n372,0,3,\"Wiklund, Mr. Jakob Alfred\",male,18,1,0,3101267,6.4958,,S\n373,0,3,\"Beavan, Mr. William Thomas\",male,19,0,0,323951,8.05,,S\n374,0,1,\"Ringhini, Mr. Sante\",male,22,0,0,PC 17760,135.6333,,C\n375,0,3,\"Palsson, Miss. Stina Viola\",female,3,3,1,349909,21.075,,S\n376,1,1,\"Meyer, Mrs. Edgar Joseph (Leila Saks)\",female,,1,0,PC 17604,82.1708,,C\n377,1,3,\"Landergren, Miss. Aurora Adelia\",female,22,0,0,C 7077,7.25,,S\n378,0,1,\"Widener, Mr. Harry Elkins\",male,27,0,2,113503,211.5,C82,C\n379,0,3,\"Betros, Mr. Tannous\",male,20,0,0,2648,4.0125,,C\n380,0,3,\"Gustafsson, Mr. Karl Gideon\",male,19,0,0,347069,7.775,,S\n381,1,1,\"Bidois, Miss. Rosalie\",female,42,0,0,PC 17757,227.525,,C\n382,1,3,\"Nakid, Miss. Maria (\"\"Mary\"\")\",female,1,0,2,2653,15.7417,,C\n383,0,3,\"Tikkanen, Mr. Juho\",male,32,0,0,STON/O 2. 3101293,7.925,,S\n384,1,1,\"Holverson, Mrs. Alexander Oskar (Mary Aline Towner)\",female,35,1,0,113789,52,,S\n385,0,3,\"Plotcharsky, Mr. Vasil\",male,,0,0,349227,7.8958,,S\n386,0,2,\"Davies, Mr. Charles Henry\",male,18,0,0,S.O.C. 14879,73.5,,S\n387,0,3,\"Goodwin, Master. Sidney Leonard\",male,1,5,2,CA 2144,46.9,,S\n388,1,2,\"Buss, Miss. Kate\",female,36,0,0,27849,13,,S\n389,0,3,\"Sadlier, Mr. Matthew\",male,,0,0,367655,7.7292,,Q\n390,1,2,\"Lehmann, Miss. Bertha\",female,17,0,0,SC 1748,12,,C\n391,1,1,\"Carter, Mr. William Ernest\",male,36,1,2,113760,120,B96 B98,S\n392,1,3,\"Jansson, Mr. Carl Olof\",male,21,0,0,350034,7.7958,,S\n393,0,3,\"Gustafsson, Mr. Johan Birger\",male,28,2,0,3101277,7.925,,S\n394,1,1,\"Newell, Miss. Marjorie\",female,23,1,0,35273,113.275,D36,C\n395,1,3,\"Sandstrom, Mrs. Hjalmar (Agnes Charlotta Bengtsson)\",female,24,0,2,PP 9549,16.7,G6,S\n396,0,3,\"Johansson, Mr. Erik\",male,22,0,0,350052,7.7958,,S\n397,0,3,\"Olsson, Miss. Elina\",female,31,0,0,350407,7.8542,,S\n398,0,2,\"McKane, Mr. Peter David\",male,46,0,0,28403,26,,S\n399,0,2,\"Pain, Dr. Alfred\",male,23,0,0,244278,10.5,,S\n400,1,2,\"Trout, Mrs. William H (Jessie L)\",female,28,0,0,240929,12.65,,S\n401,1,3,\"Niskanen, Mr. Juha\",male,39,0,0,STON/O 2. 3101289,7.925,,S\n402,0,3,\"Adams, Mr. John\",male,26,0,0,341826,8.05,,S\n403,0,3,\"Jussila, Miss. Mari Aina\",female,21,1,0,4137,9.825,,S\n404,0,3,\"Hakkarainen, Mr. Pekka Pietari\",male,28,1,0,STON/O2. 3101279,15.85,,S\n405,0,3,\"Oreskovic, Miss. Marija\",female,20,0,0,315096,8.6625,,S\n406,0,2,\"Gale, Mr. Shadrach\",male,34,1,0,28664,21,,S\n407,0,3,\"Widegren, Mr. Carl/Charles Peter\",male,51,0,0,347064,7.75,,S\n408,1,2,\"Richards, Master. William Rowe\",male,3,1,1,29106,18.75,,S\n409,0,3,\"Birkeland, Mr. Hans Martin Monsen\",male,21,0,0,312992,7.775,,S\n410,0,3,\"Lefebre, Miss. Ida\",female,,3,1,4133,25.4667,,S\n411,0,3,\"Sdycoff, Mr. Todor\",male,,0,0,349222,7.8958,,S\n412,0,3,\"Hart, Mr. Henry\",male,,0,0,394140,6.8583,,Q\n413,1,1,\"Minahan, Miss. Daisy E\",female,33,1,0,19928,90,C78,Q\n414,0,2,\"Cunningham, Mr. Alfred Fleming\",male,,0,0,239853,0,,S\n415,1,3,\"Sundman, Mr. Johan Julian\",male,44,0,0,STON/O 2. 3101269,7.925,,S\n416,0,3,\"Meek, Mrs. Thomas (Annie Louise Rowley)\",female,,0,0,343095,8.05,,S\n417,1,2,\"Drew, Mrs. James Vivian (Lulu Thorne Christian)\",female,34,1,1,28220,32.5,,S\n418,1,2,\"Silven, Miss. Lyyli Karoliina\",female,18,0,2,250652,13,,S\n419,0,2,\"Matthews, Mr. William John\",male,30,0,0,28228,13,,S\n420,0,3,\"Van Impe, Miss. Catharina\",female,10,0,2,345773,24.15,,S\n421,0,3,\"Gheorgheff, Mr. Stanio\",male,,0,0,349254,7.8958,,C\n422,0,3,\"Charters, Mr. David\",male,21,0,0,A/5. 13032,7.7333,,Q\n423,0,3,\"Zimmerman, Mr. Leo\",male,29,0,0,315082,7.875,,S\n424,0,3,\"Danbom, Mrs. Ernst Gilbert (Anna Sigrid Maria Brogren)\",female,28,1,1,347080,14.4,,S\n425,0,3,\"Rosblom, Mr. Viktor Richard\",male,18,1,1,370129,20.2125,,S\n426,0,3,\"Wiseman, Mr. Phillippe\",male,,0,0,A/4. 34244,7.25,,S\n427,1,2,\"Clarke, Mrs. Charles V (Ada Maria Winfield)\",female,28,1,0,2003,26,,S\n428,1,2,\"Phillips, Miss. Kate Florence (\"\"Mrs Kate Louise Phillips Marshall\"\")\",female,19,0,0,250655,26,,S\n429,0,3,\"Flynn, Mr. James\",male,,0,0,364851,7.75,,Q\n430,1,3,\"Pickard, Mr. Berk (Berk Trembisky)\",male,32,0,0,SOTON/O.Q. 392078,8.05,E10,S\n431,1,1,\"Bjornstrom-Steffansson, Mr. Mauritz Hakan\",male,28,0,0,110564,26.55,C52,S\n432,1,3,\"Thorneycroft, Mrs. Percival (Florence Kate White)\",female,,1,0,376564,16.1,,S\n433,1,2,\"Louch, Mrs. Charles Alexander (Alice Adelaide Slow)\",female,42,1,0,SC/AH 3085,26,,S\n434,0,3,\"Kallio, Mr. Nikolai Erland\",male,17,0,0,STON/O 2. 3101274,7.125,,S\n435,0,1,\"Silvey, Mr. William Baird\",male,50,1,0,13507,55.9,E44,S\n436,1,1,\"Carter, Miss. Lucile Polk\",female,14,1,2,113760,120,B96 B98,S\n437,0,3,\"Ford, Miss. Doolina Margaret \"\"Daisy\"\"\",female,21,2,2,W./C. 6608,34.375,,S\n438,1,2,\"Richards, Mrs. Sidney (Emily Hocking)\",female,24,2,3,29106,18.75,,S\n439,0,1,\"Fortune, Mr. Mark\",male,64,1,4,19950,263,C23 C25 C27,S\n440,0,2,\"Kvillner, Mr. Johan Henrik Johannesson\",male,31,0,0,C.A. 18723,10.5,,S\n441,1,2,\"Hart, Mrs. Benjamin (Esther Ada Bloomfield)\",female,45,1,1,F.C.C. 13529,26.25,,S\n442,0,3,\"Hampe, Mr. Leon\",male,20,0,0,345769,9.5,,S\n443,0,3,\"Petterson, Mr. Johan Emil\",male,25,1,0,347076,7.775,,S\n444,1,2,\"Reynaldo, Ms. Encarnacion\",female,28,0,0,230434,13,,S\n445,1,3,\"Johannesen-Bratthammer, Mr. Bernt\",male,,0,0,65306,8.1125,,S\n446,1,1,\"Dodge, Master. Washington\",male,4,0,2,33638,81.8583,A34,S\n447,1,2,\"Mellinger, Miss. Madeleine Violet\",female,13,0,1,250644,19.5,,S\n448,1,1,\"Seward, Mr. Frederic Kimber\",male,34,0,0,113794,26.55,,S\n449,1,3,\"Baclini, Miss. Marie Catherine\",female,5,2,1,2666,19.2583,,C\n450,1,1,\"Peuchen, Major. Arthur Godfrey\",male,52,0,0,113786,30.5,C104,S\n451,0,2,\"West, Mr. Edwy Arthur\",male,36,1,2,C.A. 34651,27.75,,S\n452,0,3,\"Hagland, Mr. Ingvald Olai Olsen\",male,,1,0,65303,19.9667,,S\n453,0,1,\"Foreman, Mr. Benjamin Laventall\",male,30,0,0,113051,27.75,C111,C\n454,1,1,\"Goldenberg, Mr. Samuel L\",male,49,1,0,17453,89.1042,C92,C\n455,0,3,\"Peduzzi, Mr. Joseph\",male,,0,0,A/5 2817,8.05,,S\n456,1,3,\"Jalsevac, Mr. Ivan\",male,29,0,0,349240,7.8958,,C\n457,0,1,\"Millet, Mr. Francis Davis\",male,65,0,0,13509,26.55,E38,S\n458,1,1,\"Kenyon, Mrs. Frederick R (Marion)\",female,,1,0,17464,51.8625,D21,S\n459,1,2,\"Toomey, Miss. Ellen\",female,50,0,0,F.C.C. 13531,10.5,,S\n460,0,3,\"O'Connor, Mr. Maurice\",male,,0,0,371060,7.75,,Q\n461,1,1,\"Anderson, Mr. Harry\",male,48,0,0,19952,26.55,E12,S\n462,0,3,\"Morley, Mr. William\",male,34,0,0,364506,8.05,,S\n463,0,1,\"Gee, Mr. Arthur H\",male,47,0,0,111320,38.5,E63,S\n464,0,2,\"Milling, Mr. Jacob Christian\",male,48,0,0,234360,13,,S\n465,0,3,\"Maisner, Mr. Simon\",male,,0,0,A/S 2816,8.05,,S\n466,0,3,\"Goncalves, Mr. Manuel Estanslas\",male,38,0,0,SOTON/O.Q. 3101306,7.05,,S\n467,0,2,\"Campbell, Mr. William\",male,,0,0,239853,0,,S\n468,0,1,\"Smart, Mr. John Montgomery\",male,56,0,0,113792,26.55,,S\n469,0,3,\"Scanlan, Mr. James\",male,,0,0,36209,7.725,,Q\n470,1,3,\"Baclini, Miss. Helene Barbara\",female,0.75,2,1,2666,19.2583,,C\n471,0,3,\"Keefe, Mr. Arthur\",male,,0,0,323592,7.25,,S\n472,0,3,\"Cacic, Mr. Luka\",male,38,0,0,315089,8.6625,,S\n473,1,2,\"West, Mrs. Edwy Arthur (Ada Mary Worth)\",female,33,1,2,C.A. 34651,27.75,,S\n474,1,2,\"Jerwan, Mrs. Amin S (Marie Marthe Thuillard)\",female,23,0,0,SC/AH Basle 541,13.7917,D,C\n475,0,3,\"Strandberg, Miss. Ida Sofia\",female,22,0,0,7553,9.8375,,S\n476,0,1,\"Clifford, Mr. George Quincy\",male,,0,0,110465,52,A14,S\n477,0,2,\"Renouf, Mr. Peter Henry\",male,34,1,0,31027,21,,S\n478,0,3,\"Braund, Mr. Lewis Richard\",male,29,1,0,3460,7.0458,,S\n479,0,3,\"Karlsson, Mr. Nils August\",male,22,0,0,350060,7.5208,,S\n480,1,3,\"Hirvonen, Miss. Hildur E\",female,2,0,1,3101298,12.2875,,S\n481,0,3,\"Goodwin, Master. Harold Victor\",male,9,5,2,CA 2144,46.9,,S\n482,0,2,\"Frost, Mr. Anthony Wood \"\"Archie\"\"\",male,,0,0,239854,0,,S\n483,0,3,\"Rouse, Mr. Richard Henry\",male,50,0,0,A/5 3594,8.05,,S\n484,1,3,\"Turkula, Mrs. (Hedwig)\",female,63,0,0,4134,9.5875,,S\n485,1,1,\"Bishop, Mr. Dickinson H\",male,25,1,0,11967,91.0792,B49,C\n486,0,3,\"Lefebre, Miss. Jeannie\",female,,3,1,4133,25.4667,,S\n487,1,1,\"Hoyt, Mrs. Frederick Maxfield (Jane Anne Forby)\",female,35,1,0,19943,90,C93,S\n488,0,1,\"Kent, Mr. Edward Austin\",male,58,0,0,11771,29.7,B37,C\n489,0,3,\"Somerton, Mr. Francis William\",male,30,0,0,A.5. 18509,8.05,,S\n490,1,3,\"Coutts, Master. Eden Leslie \"\"Neville\"\"\",male,9,1,1,C.A. 37671,15.9,,S\n491,0,3,\"Hagland, Mr. Konrad Mathias Reiersen\",male,,1,0,65304,19.9667,,S\n492,0,3,\"Windelov, Mr. Einar\",male,21,0,0,SOTON/OQ 3101317,7.25,,S\n493,0,1,\"Molson, Mr. Harry Markland\",male,55,0,0,113787,30.5,C30,S\n494,0,1,\"Artagaveytia, Mr. Ramon\",male,71,0,0,PC 17609,49.5042,,C\n495,0,3,\"Stanley, Mr. Edward Roland\",male,21,0,0,A/4 45380,8.05,,S\n496,0,3,\"Yousseff, Mr. Gerious\",male,,0,0,2627,14.4583,,C\n497,1,1,\"Eustis, Miss. Elizabeth Mussey\",female,54,1,0,36947,78.2667,D20,C\n498,0,3,\"Shellard, Mr. Frederick William\",male,,0,0,C.A. 6212,15.1,,S\n499,0,1,\"Allison, Mrs. Hudson J C (Bessie Waldo Daniels)\",female,25,1,2,113781,151.55,C22 C26,S\n500,0,3,\"Svensson, Mr. Olof\",male,24,0,0,350035,7.7958,,S\n501,0,3,\"Calic, Mr. Petar\",male,17,0,0,315086,8.6625,,S\n502,0,3,\"Canavan, Miss. Mary\",female,21,0,0,364846,7.75,,Q\n503,0,3,\"O'Sullivan, Miss. Bridget Mary\",female,,0,0,330909,7.6292,,Q\n504,0,3,\"Laitinen, Miss. Kristina Sofia\",female,37,0,0,4135,9.5875,,S\n505,1,1,\"Maioni, Miss. Roberta\",female,16,0,0,110152,86.5,B79,S\n506,0,1,\"Penasco y Castellana, Mr. Victor de Satode\",male,18,1,0,PC 17758,108.9,C65,C\n507,1,2,\"Quick, Mrs. Frederick Charles (Jane Richards)\",female,33,0,2,26360,26,,S\n508,1,1,\"Bradley, Mr. George (\"\"George Arthur Brayton\"\")\",male,,0,0,111427,26.55,,S\n509,0,3,\"Olsen, Mr. Henry Margido\",male,28,0,0,C 4001,22.525,,S\n510,1,3,\"Lang, Mr. Fang\",male,26,0,0,1601,56.4958,,S\n511,1,3,\"Daly, Mr. Eugene Patrick\",male,29,0,0,382651,7.75,,Q\n512,0,3,\"Webber, Mr. James\",male,,0,0,SOTON/OQ 3101316,8.05,,S\n513,1,1,\"McGough, Mr. James Robert\",male,36,0,0,PC 17473,26.2875,E25,S\n514,1,1,\"Rothschild, Mrs. Martin (Elizabeth L. Barrett)\",female,54,1,0,PC 17603,59.4,,C\n515,0,3,\"Coleff, Mr. Satio\",male,24,0,0,349209,7.4958,,S\n516,0,1,\"Walker, Mr. William Anderson\",male,47,0,0,36967,34.0208,D46,S\n517,1,2,\"Lemore, Mrs. (Amelia Milley)\",female,34,0,0,C.A. 34260,10.5,F33,S\n518,0,3,\"Ryan, Mr. Patrick\",male,,0,0,371110,24.15,,Q\n519,1,2,\"Angle, Mrs. William A (Florence \"\"Mary\"\" Agnes Hughes)\",female,36,1,0,226875,26,,S\n520,0,3,\"Pavlovic, Mr. Stefo\",male,32,0,0,349242,7.8958,,S\n521,1,1,\"Perreault, Miss. Anne\",female,30,0,0,12749,93.5,B73,S\n522,0,3,\"Vovk, Mr. Janko\",male,22,0,0,349252,7.8958,,S\n523,0,3,\"Lahoud, Mr. Sarkis\",male,,0,0,2624,7.225,,C\n524,1,1,\"Hippach, Mrs. Louis Albert (Ida Sophia Fischer)\",female,44,0,1,111361,57.9792,B18,C\n525,0,3,\"Kassem, Mr. Fared\",male,,0,0,2700,7.2292,,C\n526,0,3,\"Farrell, Mr. James\",male,40.5,0,0,367232,7.75,,Q\n527,1,2,\"Ridsdale, Miss. Lucy\",female,50,0,0,W./C. 14258,10.5,,S\n528,0,1,\"Farthing, Mr. John\",male,,0,0,PC 17483,221.7792,C95,S\n529,0,3,\"Salonen, Mr. Johan Werner\",male,39,0,0,3101296,7.925,,S\n530,0,2,\"Hocking, Mr. Richard George\",male,23,2,1,29104,11.5,,S\n531,1,2,\"Quick, Miss. Phyllis May\",female,2,1,1,26360,26,,S\n532,0,3,\"Toufik, Mr. Nakli\",male,,0,0,2641,7.2292,,C\n533,0,3,\"Elias, Mr. Joseph Jr\",male,17,1,1,2690,7.2292,,C\n534,1,3,\"Peter, Mrs. Catherine (Catherine Rizk)\",female,,0,2,2668,22.3583,,C\n535,0,3,\"Cacic, Miss. Marija\",female,30,0,0,315084,8.6625,,S\n536,1,2,\"Hart, Miss. Eva Miriam\",female,7,0,2,F.C.C. 13529,26.25,,S\n537,0,1,\"Butt, Major. Archibald Willingham\",male,45,0,0,113050,26.55,B38,S\n538,1,1,\"LeRoy, Miss. Bertha\",female,30,0,0,PC 17761,106.425,,C\n539,0,3,\"Risien, Mr. Samuel Beard\",male,,0,0,364498,14.5,,S\n540,1,1,\"Frolicher, Miss. Hedwig Margaritha\",female,22,0,2,13568,49.5,B39,C\n541,1,1,\"Crosby, Miss. Harriet R\",female,36,0,2,WE/P 5735,71,B22,S\n542,0,3,\"Andersson, Miss. Ingeborg Constanzia\",female,9,4,2,347082,31.275,,S\n543,0,3,\"Andersson, Miss. Sigrid Elisabeth\",female,11,4,2,347082,31.275,,S\n544,1,2,\"Beane, Mr. Edward\",male,32,1,0,2908,26,,S\n545,0,1,\"Douglas, Mr. Walter Donald\",male,50,1,0,PC 17761,106.425,C86,C\n546,0,1,\"Nicholson, Mr. Arthur Ernest\",male,64,0,0,693,26,,S\n547,1,2,\"Beane, Mrs. Edward (Ethel Clarke)\",female,19,1,0,2908,26,,S\n548,1,2,\"Padro y Manent, Mr. Julian\",male,,0,0,SC/PARIS 2146,13.8625,,C\n549,0,3,\"Goldsmith, Mr. Frank John\",male,33,1,1,363291,20.525,,S\n550,1,2,\"Davies, Master. John Morgan Jr\",male,8,1,1,C.A. 33112,36.75,,S\n551,1,1,\"Thayer, Mr. John Borland Jr\",male,17,0,2,17421,110.8833,C70,C\n552,0,2,\"Sharp, Mr. Percival James R\",male,27,0,0,244358,26,,S\n553,0,3,\"O'Brien, Mr. Timothy\",male,,0,0,330979,7.8292,,Q\n554,1,3,\"Leeni, Mr. Fahim (\"\"Philip Zenni\"\")\",male,22,0,0,2620,7.225,,C\n555,1,3,\"Ohman, Miss. Velin\",female,22,0,0,347085,7.775,,S\n556,0,1,\"Wright, Mr. George\",male,62,0,0,113807,26.55,,S\n557,1,1,\"Duff Gordon, Lady. (Lucille Christiana Sutherland) (\"\"Mrs Morgan\"\")\",female,48,1,0,11755,39.6,A16,C\n558,0,1,\"Robbins, Mr. Victor\",male,,0,0,PC 17757,227.525,,C\n559,1,1,\"Taussig, Mrs. Emil (Tillie Mandelbaum)\",female,39,1,1,110413,79.65,E67,S\n560,1,3,\"de Messemaeker, Mrs. Guillaume Joseph (Emma)\",female,36,1,0,345572,17.4,,S\n561,0,3,\"Morrow, Mr. Thomas Rowan\",male,,0,0,372622,7.75,,Q\n562,0,3,\"Sivic, Mr. Husein\",male,40,0,0,349251,7.8958,,S\n563,0,2,\"Norman, Mr. Robert Douglas\",male,28,0,0,218629,13.5,,S\n564,0,3,\"Simmons, Mr. John\",male,,0,0,SOTON/OQ 392082,8.05,,S\n565,0,3,\"Meanwell, Miss. (Marion Ogden)\",female,,0,0,SOTON/O.Q. 392087,8.05,,S\n566,0,3,\"Davies, Mr. Alfred J\",male,24,2,0,A/4 48871,24.15,,S\n567,0,3,\"Stoytcheff, Mr. Ilia\",male,19,0,0,349205,7.8958,,S\n568,0,3,\"Palsson, Mrs. Nils (Alma Cornelia Berglund)\",female,29,0,4,349909,21.075,,S\n569,0,3,\"Doharr, Mr. Tannous\",male,,0,0,2686,7.2292,,C\n570,1,3,\"Jonsson, Mr. Carl\",male,32,0,0,350417,7.8542,,S\n571,1,2,\"Harris, Mr. George\",male,62,0,0,S.W./PP 752,10.5,,S\n572,1,1,\"Appleton, Mrs. Edward Dale (Charlotte Lamson)\",female,53,2,0,11769,51.4792,C101,S\n573,1,1,\"Flynn, Mr. John Irwin (\"\"Irving\"\")\",male,36,0,0,PC 17474,26.3875,E25,S\n574,1,3,\"Kelly, Miss. Mary\",female,,0,0,14312,7.75,,Q\n575,0,3,\"Rush, Mr. Alfred George John\",male,16,0,0,A/4. 20589,8.05,,S\n576,0,3,\"Patchett, Mr. George\",male,19,0,0,358585,14.5,,S\n577,1,2,\"Garside, Miss. Ethel\",female,34,0,0,243880,13,,S\n578,1,1,\"Silvey, Mrs. William Baird (Alice Munger)\",female,39,1,0,13507,55.9,E44,S\n579,0,3,\"Caram, Mrs. Joseph (Maria Elias)\",female,,1,0,2689,14.4583,,C\n580,1,3,\"Jussila, Mr. Eiriik\",male,32,0,0,STON/O 2. 3101286,7.925,,S\n581,1,2,\"Christy, Miss. Julie Rachel\",female,25,1,1,237789,30,,S\n582,1,1,\"Thayer, Mrs. John Borland (Marian Longstreth Morris)\",female,39,1,1,17421,110.8833,C68,C\n583,0,2,\"Downton, Mr. William James\",male,54,0,0,28403,26,,S\n584,0,1,\"Ross, Mr. John Hugo\",male,36,0,0,13049,40.125,A10,C\n585,0,3,\"Paulner, Mr. Uscher\",male,,0,0,3411,8.7125,,C\n586,1,1,\"Taussig, Miss. Ruth\",female,18,0,2,110413,79.65,E68,S\n587,0,2,\"Jarvis, Mr. John Denzil\",male,47,0,0,237565,15,,S\n588,1,1,\"Frolicher-Stehli, Mr. Maxmillian\",male,60,1,1,13567,79.2,B41,C\n589,0,3,\"Gilinski, Mr. Eliezer\",male,22,0,0,14973,8.05,,S\n590,0,3,\"Murdlin, Mr. Joseph\",male,,0,0,A./5. 3235,8.05,,S\n591,0,3,\"Rintamaki, Mr. Matti\",male,35,0,0,STON/O 2. 3101273,7.125,,S\n592,1,1,\"Stephenson, Mrs. Walter Bertram (Martha Eustis)\",female,52,1,0,36947,78.2667,D20,C\n593,0,3,\"Elsbury, Mr. William James\",male,47,0,0,A/5 3902,7.25,,S\n594,0,3,\"Bourke, Miss. Mary\",female,,0,2,364848,7.75,,Q\n595,0,2,\"Chapman, Mr. John Henry\",male,37,1,0,SC/AH 29037,26,,S\n596,0,3,\"Van Impe, Mr. Jean Baptiste\",male,36,1,1,345773,24.15,,S\n597,1,2,\"Leitch, Miss. Jessie Wills\",female,,0,0,248727,33,,S\n598,0,3,\"Johnson, Mr. Alfred\",male,49,0,0,LINE,0,,S\n599,0,3,\"Boulos, Mr. Hanna\",male,,0,0,2664,7.225,,C\n600,1,1,\"Duff Gordon, Sir. Cosmo Edmund (\"\"Mr Morgan\"\")\",male,49,1,0,PC 17485,56.9292,A20,C\n601,1,2,\"Jacobsohn, Mrs. Sidney Samuel (Amy Frances Christy)\",female,24,2,1,243847,27,,S\n602,0,3,\"Slabenoff, Mr. Petco\",male,,0,0,349214,7.8958,,S\n603,0,1,\"Harrington, Mr. Charles H\",male,,0,0,113796,42.4,,S\n604,0,3,\"Torber, Mr. Ernst William\",male,44,0,0,364511,8.05,,S\n605,1,1,\"Homer, Mr. Harry (\"\"Mr E Haven\"\")\",male,35,0,0,111426,26.55,,C\n606,0,3,\"Lindell, Mr. Edvard Bengtsson\",male,36,1,0,349910,15.55,,S\n607,0,3,\"Karaic, Mr. Milan\",male,30,0,0,349246,7.8958,,S\n608,1,1,\"Daniel, Mr. Robert Williams\",male,27,0,0,113804,30.5,,S\n609,1,2,\"Laroche, Mrs. Joseph (Juliette Marie Louise Lafargue)\",female,22,1,2,SC/Paris 2123,41.5792,,C\n610,1,1,\"Shutes, Miss. Elizabeth W\",female,40,0,0,PC 17582,153.4625,C125,S\n611,0,3,\"Andersson, Mrs. Anders Johan (Alfrida Konstantia Brogren)\",female,39,1,5,347082,31.275,,S\n612,0,3,\"Jardin, Mr. Jose Neto\",male,,0,0,SOTON/O.Q. 3101305,7.05,,S\n613,1,3,\"Murphy, Miss. Margaret Jane\",female,,1,0,367230,15.5,,Q\n614,0,3,\"Horgan, Mr. John\",male,,0,0,370377,7.75,,Q\n615,0,3,\"Brocklebank, Mr. William Alfred\",male,35,0,0,364512,8.05,,S\n616,1,2,\"Herman, Miss. Alice\",female,24,1,2,220845,65,,S\n617,0,3,\"Danbom, Mr. Ernst Gilbert\",male,34,1,1,347080,14.4,,S\n618,0,3,\"Lobb, Mrs. William Arthur (Cordelia K Stanlick)\",female,26,1,0,A/5. 3336,16.1,,S\n619,1,2,\"Becker, Miss. Marion Louise\",female,4,2,1,230136,39,F4,S\n620,0,2,\"Gavey, Mr. Lawrence\",male,26,0,0,31028,10.5,,S\n621,0,3,\"Yasbeck, Mr. Antoni\",male,27,1,0,2659,14.4542,,C\n622,1,1,\"Kimball, Mr. Edwin Nelson Jr\",male,42,1,0,11753,52.5542,D19,S\n623,1,3,\"Nakid, Mr. Sahid\",male,20,1,1,2653,15.7417,,C\n624,0,3,\"Hansen, Mr. Henry Damsgaard\",male,21,0,0,350029,7.8542,,S\n625,0,3,\"Bowen, Mr. David John \"\"Dai\"\"\",male,21,0,0,54636,16.1,,S\n626,0,1,\"Sutton, Mr. Frederick\",male,61,0,0,36963,32.3208,D50,S\n627,0,2,\"Kirkland, Rev. Charles Leonard\",male,57,0,0,219533,12.35,,Q\n628,1,1,\"Longley, Miss. Gretchen Fiske\",female,21,0,0,13502,77.9583,D9,S\n629,0,3,\"Bostandyeff, Mr. Guentcho\",male,26,0,0,349224,7.8958,,S\n630,0,3,\"O'Connell, Mr. Patrick D\",male,,0,0,334912,7.7333,,Q\n631,1,1,\"Barkworth, Mr. Algernon Henry Wilson\",male,80,0,0,27042,30,A23,S\n632,0,3,\"Lundahl, Mr. Johan Svensson\",male,51,0,0,347743,7.0542,,S\n633,1,1,\"Stahelin-Maeglin, Dr. Max\",male,32,0,0,13214,30.5,B50,C\n634,0,1,\"Parr, Mr. William Henry Marsh\",male,,0,0,112052,0,,S\n635,0,3,\"Skoog, Miss. Mabel\",female,9,3,2,347088,27.9,,S\n636,1,2,\"Davis, Miss. Mary\",female,28,0,0,237668,13,,S\n637,0,3,\"Leinonen, Mr. Antti Gustaf\",male,32,0,0,STON/O 2. 3101292,7.925,,S\n638,0,2,\"Collyer, Mr. Harvey\",male,31,1,1,C.A. 31921,26.25,,S\n639,0,3,\"Panula, Mrs. Juha (Maria Emilia Ojala)\",female,41,0,5,3101295,39.6875,,S\n640,0,3,\"Thorneycroft, Mr. Percival\",male,,1,0,376564,16.1,,S\n641,0,3,\"Jensen, Mr. Hans Peder\",male,20,0,0,350050,7.8542,,S\n642,1,1,\"Sagesser, Mlle. Emma\",female,24,0,0,PC 17477,69.3,B35,C\n643,0,3,\"Skoog, Miss. Margit Elizabeth\",female,2,3,2,347088,27.9,,S\n644,1,3,\"Foo, Mr. Choong\",male,,0,0,1601,56.4958,,S\n645,1,3,\"Baclini, Miss. Eugenie\",female,0.75,2,1,2666,19.2583,,C\n646,1,1,\"Harper, Mr. Henry Sleeper\",male,48,1,0,PC 17572,76.7292,D33,C\n647,0,3,\"Cor, Mr. Liudevit\",male,19,0,0,349231,7.8958,,S\n648,1,1,\"Simonius-Blumer, Col. Oberst Alfons\",male,56,0,0,13213,35.5,A26,C\n649,0,3,\"Willey, Mr. Edward\",male,,0,0,S.O./P.P. 751,7.55,,S\n650,1,3,\"Stanley, Miss. Amy Zillah Elsie\",female,23,0,0,CA. 2314,7.55,,S\n651,0,3,\"Mitkoff, Mr. Mito\",male,,0,0,349221,7.8958,,S\n652,1,2,\"Doling, Miss. Elsie\",female,18,0,1,231919,23,,S\n653,0,3,\"Kalvik, Mr. Johannes Halvorsen\",male,21,0,0,8475,8.4333,,S\n654,1,3,\"O'Leary, Miss. Hanora \"\"Norah\"\"\",female,,0,0,330919,7.8292,,Q\n655,0,3,\"Hegarty, Miss. Hanora \"\"Nora\"\"\",female,18,0,0,365226,6.75,,Q\n656,0,2,\"Hickman, Mr. Leonard Mark\",male,24,2,0,S.O.C. 14879,73.5,,S\n657,0,3,\"Radeff, Mr. Alexander\",male,,0,0,349223,7.8958,,S\n658,0,3,\"Bourke, Mrs. John (Catherine)\",female,32,1,1,364849,15.5,,Q\n659,0,2,\"Eitemiller, Mr. George Floyd\",male,23,0,0,29751,13,,S\n660,0,1,\"Newell, Mr. Arthur Webster\",male,58,0,2,35273,113.275,D48,C\n661,1,1,\"Frauenthal, Dr. Henry William\",male,50,2,0,PC 17611,133.65,,S\n662,0,3,\"Badt, Mr. Mohamed\",male,40,0,0,2623,7.225,,C\n663,0,1,\"Colley, Mr. Edward Pomeroy\",male,47,0,0,5727,25.5875,E58,S\n664,0,3,\"Coleff, Mr. Peju\",male,36,0,0,349210,7.4958,,S\n665,1,3,\"Lindqvist, Mr. Eino William\",male,20,1,0,STON/O 2. 3101285,7.925,,S\n666,0,2,\"Hickman, Mr. Lewis\",male,32,2,0,S.O.C. 14879,73.5,,S\n667,0,2,\"Butler, Mr. Reginald Fenton\",male,25,0,0,234686,13,,S\n668,0,3,\"Rommetvedt, Mr. Knud Paust\",male,,0,0,312993,7.775,,S\n669,0,3,\"Cook, Mr. Jacob\",male,43,0,0,A/5 3536,8.05,,S\n670,1,1,\"Taylor, Mrs. Elmer Zebley (Juliet Cummins Wright)\",female,,1,0,19996,52,C126,S\n671,1,2,\"Brown, Mrs. Thomas William Solomon (Elizabeth Catherine Ford)\",female,40,1,1,29750,39,,S\n672,0,1,\"Davidson, Mr. Thornton\",male,31,1,0,F.C. 12750,52,B71,S\n673,0,2,\"Mitchell, Mr. Henry Michael\",male,70,0,0,C.A. 24580,10.5,,S\n674,1,2,\"Wilhelms, Mr. Charles\",male,31,0,0,244270,13,,S\n675,0,2,\"Watson, Mr. Ennis Hastings\",male,,0,0,239856,0,,S\n676,0,3,\"Edvardsson, Mr. Gustaf Hjalmar\",male,18,0,0,349912,7.775,,S\n677,0,3,\"Sawyer, Mr. Frederick Charles\",male,24.5,0,0,342826,8.05,,S\n678,1,3,\"Turja, Miss. Anna Sofia\",female,18,0,0,4138,9.8417,,S\n679,0,3,\"Goodwin, Mrs. Frederick (Augusta Tyler)\",female,43,1,6,CA 2144,46.9,,S\n680,1,1,\"Cardeza, Mr. Thomas Drake Martinez\",male,36,0,1,PC 17755,512.3292,B51 B53 B55,C\n681,0,3,\"Peters, Miss. Katie\",female,,0,0,330935,8.1375,,Q\n682,1,1,\"Hassab, Mr. Hammad\",male,27,0,0,PC 17572,76.7292,D49,C\n683,0,3,\"Olsvigen, Mr. Thor Anderson\",male,20,0,0,6563,9.225,,S\n684,0,3,\"Goodwin, Mr. Charles Edward\",male,14,5,2,CA 2144,46.9,,S\n685,0,2,\"Brown, Mr. Thomas William Solomon\",male,60,1,1,29750,39,,S\n686,0,2,\"Laroche, Mr. Joseph Philippe Lemercier\",male,25,1,2,SC/Paris 2123,41.5792,,C\n687,0,3,\"Panula, Mr. Jaako Arnold\",male,14,4,1,3101295,39.6875,,S\n688,0,3,\"Dakic, Mr. Branko\",male,19,0,0,349228,10.1708,,S\n689,0,3,\"Fischer, Mr. Eberhard Thelander\",male,18,0,0,350036,7.7958,,S\n690,1,1,\"Madill, Miss. Georgette Alexandra\",female,15,0,1,24160,211.3375,B5,S\n691,1,1,\"Dick, Mr. Albert Adrian\",male,31,1,0,17474,57,B20,S\n692,1,3,\"Karun, Miss. Manca\",female,4,0,1,349256,13.4167,,C\n693,1,3,\"Lam, Mr. Ali\",male,,0,0,1601,56.4958,,S\n694,0,3,\"Saad, Mr. Khalil\",male,25,0,0,2672,7.225,,C\n695,0,1,\"Weir, Col. John\",male,60,0,0,113800,26.55,,S\n696,0,2,\"Chapman, Mr. Charles Henry\",male,52,0,0,248731,13.5,,S\n697,0,3,\"Kelly, Mr. James\",male,44,0,0,363592,8.05,,S\n698,1,3,\"Mullens, Miss. Katherine \"\"Katie\"\"\",female,,0,0,35852,7.7333,,Q\n699,0,1,\"Thayer, Mr. John Borland\",male,49,1,1,17421,110.8833,C68,C\n700,0,3,\"Humblen, Mr. Adolf Mathias Nicolai Olsen\",male,42,0,0,348121,7.65,F G63,S\n701,1,1,\"Astor, Mrs. John Jacob (Madeleine Talmadge Force)\",female,18,1,0,PC 17757,227.525,C62 C64,C\n702,1,1,\"Silverthorne, Mr. Spencer Victor\",male,35,0,0,PC 17475,26.2875,E24,S\n703,0,3,\"Barbara, Miss. Saiide\",female,18,0,1,2691,14.4542,,C\n704,0,3,\"Gallagher, Mr. Martin\",male,25,0,0,36864,7.7417,,Q\n705,0,3,\"Hansen, Mr. Henrik Juul\",male,26,1,0,350025,7.8542,,S\n706,0,2,\"Morley, Mr. Henry Samuel (\"\"Mr Henry Marshall\"\")\",male,39,0,0,250655,26,,S\n707,1,2,\"Kelly, Mrs. Florence \"\"Fannie\"\"\",female,45,0,0,223596,13.5,,S\n708,1,1,\"Calderhead, Mr. Edward Pennington\",male,42,0,0,PC 17476,26.2875,E24,S\n709,1,1,\"Cleaver, Miss. Alice\",female,22,0,0,113781,151.55,,S\n710,1,3,\"Moubarek, Master. Halim Gonios (\"\"William George\"\")\",male,,1,1,2661,15.2458,,C\n711,1,1,\"Mayne, Mlle. Berthe Antonine (\"\"Mrs de Villiers\"\")\",female,24,0,0,PC 17482,49.5042,C90,C\n712,0,1,\"Klaber, Mr. Herman\",male,,0,0,113028,26.55,C124,S\n713,1,1,\"Taylor, Mr. Elmer Zebley\",male,48,1,0,19996,52,C126,S\n714,0,3,\"Larsson, Mr. August Viktor\",male,29,0,0,7545,9.4833,,S\n715,0,2,\"Greenberg, Mr. Samuel\",male,52,0,0,250647,13,,S\n716,0,3,\"Soholt, Mr. Peter Andreas Lauritz Andersen\",male,19,0,0,348124,7.65,F G73,S\n717,1,1,\"Endres, Miss. Caroline Louise\",female,38,0,0,PC 17757,227.525,C45,C\n718,1,2,\"Troutt, Miss. Edwina Celia \"\"Winnie\"\"\",female,27,0,0,34218,10.5,E101,S\n719,0,3,\"McEvoy, Mr. Michael\",male,,0,0,36568,15.5,,Q\n720,0,3,\"Johnson, Mr. Malkolm Joackim\",male,33,0,0,347062,7.775,,S\n721,1,2,\"Harper, Miss. Annie Jessie \"\"Nina\"\"\",female,6,0,1,248727,33,,S\n722,0,3,\"Jensen, Mr. Svend Lauritz\",male,17,1,0,350048,7.0542,,S\n723,0,2,\"Gillespie, Mr. William Henry\",male,34,0,0,12233,13,,S\n724,0,2,\"Hodges, Mr. Henry Price\",male,50,0,0,250643,13,,S\n725,1,1,\"Chambers, Mr. Norman Campbell\",male,27,1,0,113806,53.1,E8,S\n726,0,3,\"Oreskovic, Mr. Luka\",male,20,0,0,315094,8.6625,,S\n727,1,2,\"Renouf, Mrs. Peter Henry (Lillian Jefferys)\",female,30,3,0,31027,21,,S\n728,1,3,\"Mannion, Miss. Margareth\",female,,0,0,36866,7.7375,,Q\n729,0,2,\"Bryhl, Mr. Kurt Arnold Gottfrid\",male,25,1,0,236853,26,,S\n730,0,3,\"Ilmakangas, Miss. Pieta Sofia\",female,25,1,0,STON/O2. 3101271,7.925,,S\n731,1,1,\"Allen, Miss. Elisabeth Walton\",female,29,0,0,24160,211.3375,B5,S\n732,0,3,\"Hassan, Mr. Houssein G N\",male,11,0,0,2699,18.7875,,C\n733,0,2,\"Knight, Mr. Robert J\",male,,0,0,239855,0,,S\n734,0,2,\"Berriman, Mr. William John\",male,23,0,0,28425,13,,S\n735,0,2,\"Troupiansky, Mr. Moses Aaron\",male,23,0,0,233639,13,,S\n736,0,3,\"Williams, Mr. Leslie\",male,28.5,0,0,54636,16.1,,S\n737,0,3,\"Ford, Mrs. Edward (Margaret Ann Watson)\",female,48,1,3,W./C. 6608,34.375,,S\n738,1,1,\"Lesurer, Mr. Gustave J\",male,35,0,0,PC 17755,512.3292,B101,C\n739,0,3,\"Ivanoff, Mr. Kanio\",male,,0,0,349201,7.8958,,S\n740,0,3,\"Nankoff, Mr. Minko\",male,,0,0,349218,7.8958,,S\n741,1,1,\"Hawksford, Mr. Walter James\",male,,0,0,16988,30,D45,S\n742,0,1,\"Cavendish, Mr. Tyrell William\",male,36,1,0,19877,78.85,C46,S\n743,1,1,\"Ryerson, Miss. Susan Parker \"\"Suzette\"\"\",female,21,2,2,PC 17608,262.375,B57 B59 B63 B66,C\n744,0,3,\"McNamee, Mr. Neal\",male,24,1,0,376566,16.1,,S\n745,1,3,\"Stranden, Mr. Juho\",male,31,0,0,STON/O 2. 3101288,7.925,,S\n746,0,1,\"Crosby, Capt. Edward Gifford\",male,70,1,1,WE/P 5735,71,B22,S\n747,0,3,\"Abbott, Mr. Rossmore Edward\",male,16,1,1,C.A. 2673,20.25,,S\n748,1,2,\"Sinkkonen, Miss. Anna\",female,30,0,0,250648,13,,S\n749,0,1,\"Marvin, Mr. Daniel Warner\",male,19,1,0,113773,53.1,D30,S\n750,0,3,\"Connaghton, Mr. Michael\",male,31,0,0,335097,7.75,,Q\n751,1,2,\"Wells, Miss. Joan\",female,4,1,1,29103,23,,S\n752,1,3,\"Moor, Master. Meier\",male,6,0,1,392096,12.475,E121,S\n753,0,3,\"Vande Velde, Mr. Johannes Joseph\",male,33,0,0,345780,9.5,,S\n754,0,3,\"Jonkoff, Mr. Lalio\",male,23,0,0,349204,7.8958,,S\n755,1,2,\"Herman, Mrs. Samuel (Jane Laver)\",female,48,1,2,220845,65,,S\n756,1,2,\"Hamalainen, Master. Viljo\",male,0.67,1,1,250649,14.5,,S\n757,0,3,\"Carlsson, Mr. August Sigfrid\",male,28,0,0,350042,7.7958,,S\n758,0,2,\"Bailey, Mr. Percy Andrew\",male,18,0,0,29108,11.5,,S\n759,0,3,\"Theobald, Mr. Thomas Leonard\",male,34,0,0,363294,8.05,,S\n760,1,1,\"Rothes, the Countess. of (Lucy Noel Martha Dyer-Edwards)\",female,33,0,0,110152,86.5,B77,S\n761,0,3,\"Garfirth, Mr. John\",male,,0,0,358585,14.5,,S\n762,0,3,\"Nirva, Mr. Iisakki Antino Aijo\",male,41,0,0,SOTON/O2 3101272,7.125,,S\n763,1,3,\"Barah, Mr. Hanna Assi\",male,20,0,0,2663,7.2292,,C\n764,1,1,\"Carter, Mrs. William Ernest (Lucile Polk)\",female,36,1,2,113760,120,B96 B98,S\n765,0,3,\"Eklund, Mr. Hans Linus\",male,16,0,0,347074,7.775,,S\n766,1,1,\"Hogeboom, Mrs. John C (Anna Andrews)\",female,51,1,0,13502,77.9583,D11,S\n767,0,1,\"Brewe, Dr. Arthur Jackson\",male,,0,0,112379,39.6,,C\n768,0,3,\"Mangan, Miss. Mary\",female,30.5,0,0,364850,7.75,,Q\n769,0,3,\"Moran, Mr. Daniel J\",male,,1,0,371110,24.15,,Q\n770,0,3,\"Gronnestad, Mr. Daniel Danielsen\",male,32,0,0,8471,8.3625,,S\n771,0,3,\"Lievens, Mr. Rene Aime\",male,24,0,0,345781,9.5,,S\n772,0,3,\"Jensen, Mr. Niels Peder\",male,48,0,0,350047,7.8542,,S\n773,0,2,\"Mack, Mrs. (Mary)\",female,57,0,0,S.O./P.P. 3,10.5,E77,S\n774,0,3,\"Elias, Mr. Dibo\",male,,0,0,2674,7.225,,C\n775,1,2,\"Hocking, Mrs. Elizabeth (Eliza Needs)\",female,54,1,3,29105,23,,S\n776,0,3,\"Myhrman, Mr. Pehr Fabian Oliver Malkolm\",male,18,0,0,347078,7.75,,S\n777,0,3,\"Tobin, Mr. Roger\",male,,0,0,383121,7.75,F38,Q\n778,1,3,\"Emanuel, Miss. Virginia Ethel\",female,5,0,0,364516,12.475,,S\n779,0,3,\"Kilgannon, Mr. Thomas J\",male,,0,0,36865,7.7375,,Q\n780,1,1,\"Robert, Mrs. Edward Scott (Elisabeth Walton McMillan)\",female,43,0,1,24160,211.3375,B3,S\n781,1,3,\"Ayoub, Miss. Banoura\",female,13,0,0,2687,7.2292,,C\n782,1,1,\"Dick, Mrs. Albert Adrian (Vera Gillespie)\",female,17,1,0,17474,57,B20,S\n783,0,1,\"Long, Mr. Milton Clyde\",male,29,0,0,113501,30,D6,S\n784,0,3,\"Johnston, Mr. Andrew G\",male,,1,2,W./C. 6607,23.45,,S\n785,0,3,\"Ali, Mr. William\",male,25,0,0,SOTON/O.Q. 3101312,7.05,,S\n786,0,3,\"Harmer, Mr. Abraham (David Lishin)\",male,25,0,0,374887,7.25,,S\n787,1,3,\"Sjoblom, Miss. Anna Sofia\",female,18,0,0,3101265,7.4958,,S\n788,0,3,\"Rice, Master. George Hugh\",male,8,4,1,382652,29.125,,Q\n789,1,3,\"Dean, Master. Bertram Vere\",male,1,1,2,C.A. 2315,20.575,,S\n790,0,1,\"Guggenheim, Mr. Benjamin\",male,46,0,0,PC 17593,79.2,B82 B84,C\n791,0,3,\"Keane, Mr. Andrew \"\"Andy\"\"\",male,,0,0,12460,7.75,,Q\n792,0,2,\"Gaskell, Mr. Alfred\",male,16,0,0,239865,26,,S\n793,0,3,\"Sage, Miss. Stella Anna\",female,,8,2,CA. 2343,69.55,,S\n794,0,1,\"Hoyt, Mr. William Fisher\",male,,0,0,PC 17600,30.6958,,C\n795,0,3,\"Dantcheff, Mr. Ristiu\",male,25,0,0,349203,7.8958,,S\n796,0,2,\"Otter, Mr. Richard\",male,39,0,0,28213,13,,S\n797,1,1,\"Leader, Dr. Alice (Farnham)\",female,49,0,0,17465,25.9292,D17,S\n798,1,3,\"Osman, Mrs. Mara\",female,31,0,0,349244,8.6833,,S\n799,0,3,\"Ibrahim Shawah, Mr. Yousseff\",male,30,0,0,2685,7.2292,,C\n800,0,3,\"Van Impe, Mrs. Jean Baptiste (Rosalie Paula Govaert)\",female,30,1,1,345773,24.15,,S\n801,0,2,\"Ponesell, Mr. Martin\",male,34,0,0,250647,13,,S\n802,1,2,\"Collyer, Mrs. Harvey (Charlotte Annie Tate)\",female,31,1,1,C.A. 31921,26.25,,S\n803,1,1,\"Carter, Master. William Thornton II\",male,11,1,2,113760,120,B96 B98,S\n804,1,3,\"Thomas, Master. Assad Alexander\",male,0.42,0,1,2625,8.5167,,C\n805,1,3,\"Hedman, Mr. Oskar Arvid\",male,27,0,0,347089,6.975,,S\n806,0,3,\"Johansson, Mr. Karl Johan\",male,31,0,0,347063,7.775,,S\n807,0,1,\"Andrews, Mr. Thomas Jr\",male,39,0,0,112050,0,A36,S\n808,0,3,\"Pettersson, Miss. Ellen Natalia\",female,18,0,0,347087,7.775,,S\n809,0,2,\"Meyer, Mr. August\",male,39,0,0,248723,13,,S\n810,1,1,\"Chambers, Mrs. Norman Campbell (Bertha Griggs)\",female,33,1,0,113806,53.1,E8,S\n811,0,3,\"Alexander, Mr. William\",male,26,0,0,3474,7.8875,,S\n812,0,3,\"Lester, Mr. James\",male,39,0,0,A/4 48871,24.15,,S\n813,0,2,\"Slemen, Mr. Richard James\",male,35,0,0,28206,10.5,,S\n814,0,3,\"Andersson, Miss. Ebba Iris Alfrida\",female,6,4,2,347082,31.275,,S\n815,0,3,\"Tomlin, Mr. Ernest Portage\",male,30.5,0,0,364499,8.05,,S\n816,0,1,\"Fry, Mr. Richard\",male,,0,0,112058,0,B102,S\n817,0,3,\"Heininen, Miss. Wendla Maria\",female,23,0,0,STON/O2. 3101290,7.925,,S\n818,0,2,\"Mallet, Mr. Albert\",male,31,1,1,S.C./PARIS 2079,37.0042,,C\n819,0,3,\"Holm, Mr. John Fredrik Alexander\",male,43,0,0,C 7075,6.45,,S\n820,0,3,\"Skoog, Master. Karl Thorsten\",male,10,3,2,347088,27.9,,S\n821,1,1,\"Hays, Mrs. Charles Melville (Clara Jennings Gregg)\",female,52,1,1,12749,93.5,B69,S\n822,1,3,\"Lulic, Mr. Nikola\",male,27,0,0,315098,8.6625,,S\n823,0,1,\"Reuchlin, Jonkheer. John George\",male,38,0,0,19972,0,,S\n824,1,3,\"Moor, Mrs. (Beila)\",female,27,0,1,392096,12.475,E121,S\n825,0,3,\"Panula, Master. Urho Abraham\",male,2,4,1,3101295,39.6875,,S\n826,0,3,\"Flynn, Mr. John\",male,,0,0,368323,6.95,,Q\n827,0,3,\"Lam, Mr. Len\",male,,0,0,1601,56.4958,,S\n828,1,2,\"Mallet, Master. Andre\",male,1,0,2,S.C./PARIS 2079,37.0042,,C\n829,1,3,\"McCormack, Mr. Thomas Joseph\",male,,0,0,367228,7.75,,Q\n830,1,1,\"Stone, Mrs. George Nelson (Martha Evelyn)\",female,62,0,0,113572,80,B28,\n831,1,3,\"Yasbeck, Mrs. Antoni (Selini Alexander)\",female,15,1,0,2659,14.4542,,C\n832,1,2,\"Richards, Master. George Sibley\",male,0.83,1,1,29106,18.75,,S\n833,0,3,\"Saad, Mr. Amin\",male,,0,0,2671,7.2292,,C\n834,0,3,\"Augustsson, Mr. Albert\",male,23,0,0,347468,7.8542,,S\n835,0,3,\"Allum, Mr. Owen George\",male,18,0,0,2223,8.3,,S\n836,1,1,\"Compton, Miss. Sara Rebecca\",female,39,1,1,PC 17756,83.1583,E49,C\n837,0,3,\"Pasic, Mr. Jakob\",male,21,0,0,315097,8.6625,,S\n838,0,3,\"Sirota, Mr. Maurice\",male,,0,0,392092,8.05,,S\n839,1,3,\"Chip, Mr. Chang\",male,32,0,0,1601,56.4958,,S\n840,1,1,\"Marechal, Mr. Pierre\",male,,0,0,11774,29.7,C47,C\n841,0,3,\"Alhomaki, Mr. Ilmari Rudolf\",male,20,0,0,SOTON/O2 3101287,7.925,,S\n842,0,2,\"Mudd, Mr. Thomas Charles\",male,16,0,0,S.O./P.P. 3,10.5,,S\n843,1,1,\"Serepeca, Miss. Augusta\",female,30,0,0,113798,31,,C\n844,0,3,\"Lemberopolous, Mr. Peter L\",male,34.5,0,0,2683,6.4375,,C\n845,0,3,\"Culumovic, Mr. Jeso\",male,17,0,0,315090,8.6625,,S\n846,0,3,\"Abbing, Mr. Anthony\",male,42,0,0,C.A. 5547,7.55,,S\n847,0,3,\"Sage, Mr. Douglas Bullen\",male,,8,2,CA. 2343,69.55,,S\n848,0,3,\"Markoff, Mr. Marin\",male,35,0,0,349213,7.8958,,C\n849,0,2,\"Harper, Rev. John\",male,28,0,1,248727,33,,S\n850,1,1,\"Goldenberg, Mrs. Samuel L (Edwiga Grabowska)\",female,,1,0,17453,89.1042,C92,C\n851,0,3,\"Andersson, Master. Sigvard Harald Elias\",male,4,4,2,347082,31.275,,S\n852,0,3,\"Svensson, Mr. Johan\",male,74,0,0,347060,7.775,,S\n853,0,3,\"Boulos, Miss. Nourelain\",female,9,1,1,2678,15.2458,,C\n854,1,1,\"Lines, Miss. Mary Conover\",female,16,0,1,PC 17592,39.4,D28,S\n855,0,2,\"Carter, Mrs. Ernest Courtenay (Lilian Hughes)\",female,44,1,0,244252,26,,S\n856,1,3,\"Aks, Mrs. Sam (Leah Rosen)\",female,18,0,1,392091,9.35,,S\n857,1,1,\"Wick, Mrs. George Dennick (Mary Hitchcock)\",female,45,1,1,36928,164.8667,,S\n858,1,1,\"Daly, Mr. Peter Denis \",male,51,0,0,113055,26.55,E17,S\n859,1,3,\"Baclini, Mrs. Solomon (Latifa Qurban)\",female,24,0,3,2666,19.2583,,C\n860,0,3,\"Razi, Mr. Raihed\",male,,0,0,2629,7.2292,,C\n861,0,3,\"Hansen, Mr. Claus Peter\",male,41,2,0,350026,14.1083,,S\n862,0,2,\"Giles, Mr. Frederick Edward\",male,21,1,0,28134,11.5,,S\n863,1,1,\"Swift, Mrs. Frederick Joel (Margaret Welles Barron)\",female,48,0,0,17466,25.9292,D17,S\n864,0,3,\"Sage, Miss. Dorothy Edith \"\"Dolly\"\"\",female,,8,2,CA. 2343,69.55,,S\n865,0,2,\"Gill, Mr. John William\",male,24,0,0,233866,13,,S\n866,1,2,\"Bystrom, Mrs. (Karolina)\",female,42,0,0,236852,13,,S\n867,1,2,\"Duran y More, Miss. Asuncion\",female,27,1,0,SC/PARIS 2149,13.8583,,C\n868,0,1,\"Roebling, Mr. Washington Augustus II\",male,31,0,0,PC 17590,50.4958,A24,S\n869,0,3,\"van Melkebeke, Mr. Philemon\",male,,0,0,345777,9.5,,S\n870,1,3,\"Johnson, Master. Harold Theodor\",male,4,1,1,347742,11.1333,,S\n871,0,3,\"Balkic, Mr. Cerin\",male,26,0,0,349248,7.8958,,S\n872,1,1,\"Beckwith, Mrs. Richard Leonard (Sallie Monypeny)\",female,47,1,1,11751,52.5542,D35,S\n873,0,1,\"Carlsson, Mr. Frans Olof\",male,33,0,0,695,5,B51 B53 B55,S\n874,0,3,\"Vander Cruyssen, Mr. Victor\",male,47,0,0,345765,9,,S\n875,1,2,\"Abelson, Mrs. Samuel (Hannah Wizosky)\",female,28,1,0,P/PP 3381,24,,C\n876,1,3,\"Najib, Miss. Adele Kiamie \"\"Jane\"\"\",female,15,0,0,2667,7.225,,C\n877,0,3,\"Gustafsson, Mr. Alfred Ossian\",male,20,0,0,7534,9.8458,,S\n878,0,3,\"Petroff, Mr. Nedelio\",male,19,0,0,349212,7.8958,,S\n879,0,3,\"Laleff, Mr. Kristo\",male,,0,0,349217,7.8958,,S\n880,1,1,\"Potter, Mrs. Thomas Jr (Lily Alexenia Wilson)\",female,56,0,1,11767,83.1583,C50,C\n881,1,2,\"Shelley, Mrs. William (Imanita Parrish Hall)\",female,25,0,1,230433,26,,S\n882,0,3,\"Markun, Mr. Johann\",male,33,0,0,349257,7.8958,,S\n883,0,3,\"Dahlberg, Miss. Gerda Ulrika\",female,22,0,0,7552,10.5167,,S\n884,0,2,\"Banfield, Mr. Frederick James\",male,28,0,0,C.A./SOTON 34068,10.5,,S\n885,0,3,\"Sutehall, Mr. Henry Jr\",male,25,0,0,SOTON/OQ 392076,7.05,,S\n886,0,3,\"Rice, Mrs. William (Margaret Norton)\",female,39,0,5,382652,29.125,,Q\n887,0,2,\"Montvila, Rev. Juozas\",male,27,0,0,211536,13,,S\n888,1,1,\"Graham, Miss. Margaret Edith\",female,19,0,0,112053,30,B42,S\n889,0,3,\"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\",female,,1,2,W./C. 6607,23.45,,S\n890,1,1,\"Behr, Mr. Karl Howell\",male,26,0,0,111369,30,C148,C\n891,0,3,\"Dooley, Mr. Patrick\",male,32,0,0,370376,7.75,,Q\n" }, { "path": "src/main/java/com/datastax/loader/AbstractDynamicRateLimiter.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\npublic abstract class AbstractDynamicRateLimiter extends RateLimiter {\n protected long lastCheck = 0;\n protected long howOften;\n protected double lastStat;\n protected double maxStat;\n protected double minStat;\n protected double downFraction;\n protected double upFraction;\n protected boolean invertLogic;\n\n public AbstractDynamicRateLimiter(double inRate, long inHowOften,\n double inMaxStat, double inMinStat,\n double inDownFraction, \n double inUpFraction, \n boolean inInvertLogic) {\n this(inRate, Long.MAX_VALUE, inHowOften, inMaxStat, inMinStat,\n inDownFraction, inUpFraction, inInvertLogic);\n }\n\n public AbstractDynamicRateLimiter(double inRate, long inUpdateRate,\n long inHowOften, double inMaxStat, \n double inMinStat, double inDownFraction, \n double inUpFraction, \n boolean inInvertLogic) {\n super(inRate);\n howOften = inHowOften;\n maxStat = inMaxStat;\n minStat = inMinStat;\n downFraction = inDownFraction;\n upFraction = inUpFraction;\n invertLogic = inInvertLogic;\n }\n\n public void acquire() {\n this.acquire(1);\n }\n\n public synchronized void acquire(int permits) {\n long currTime = System.currentTimeMillis();\n if (currTime - lastCheck > howOften) {\n adjustRate();\n lastCheck = currTime;\n }\n super.acquire(permits);\n }\n\n protected synchronized void adjustRate() {\n double currStat = getCurrStat();\n if (statTooHigh(currStat)) {\n if (invertLogic)\n adjustRateUp();\n else\n adjustRateDown();\n System.err.println(\"Adjusting rate down : \" + currStat + \" > \" + maxStat + \" \" + super.getRate());\n }\n else if (statTooLow(currStat)) {\n if (invertLogic)\n adjustRateDown();\n else\n adjustRateUp();\n System.err.println(\"Adjusting rate up : \" + currStat + \" > \" + maxStat + \" \" + super.getRate());\n }\n }\n\n protected synchronized boolean statTooHigh(double currStat) {\n return currStat > maxStat;\n }\n\n protected synchronized boolean statTooLow(double currStat) {\n return currStat < minStat;\n }\n\n protected synchronized void adjustRateDown() {\n double currRate = super.getRate();\n super.setRate(currRate - (currRate * downFraction));\n }\n\n protected synchronized void adjustRateUp() {\n double currRate = super.getRate();\n super.setRate(currRate + (currRate * upFraction));\n }\n\n protected abstract double getCurrStat();\n}\n" }, { "path": "src/main/java/com/datastax/loader/CqlDelimLoad.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport com.datastax.loader.parser.BooleanParser;\nimport com.datastax.loader.futures.FutureManager;\nimport com.datastax.loader.futures.PrintingFutureSet;\n\nimport java.util.List;\nimport java.util.ArrayList;\nimport java.util.Map;\nimport java.util.HashMap;\nimport java.util.Set;\nimport java.util.HashSet;\nimport java.util.Deque;\nimport java.util.ArrayDeque;\nimport java.util.Comparator;\nimport java.util.Arrays;\nimport java.util.Locale;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeUnit;\nimport java.io.File;\nimport java.io.BufferedReader;\nimport java.io.FileReader;\nimport java.io.InputStreamReader;\nimport java.io.IOException;\nimport java.io.FileOutputStream;\nimport java.io.InputStream;\nimport java.io.FileInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.PrintStream;\nimport java.io.FileNotFoundException;\nimport java.text.ParseException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.security.KeyStore;\nimport java.security.SecureRandom;\nimport java.security.KeyStoreException;\nimport java.security.KeyManagementException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.UnrecoverableKeyException;\nimport java.security.cert.CertificateException;\nimport javax.net.ssl.KeyManagerFactory;\nimport javax.net.ssl.SSLContext;\nimport javax.net.ssl.TrustManagerFactory;\n\nimport com.datastax.driver.core.Cluster;\nimport com.datastax.driver.core.Metrics;\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.ProtocolVersion;\nimport com.datastax.driver.core.PoolingOptions;\nimport com.datastax.driver.core.ProtocolOptions;\nimport com.datastax.driver.core.HostDistance;\nimport com.datastax.driver.core.PreparedStatement;\nimport com.datastax.driver.core.ConsistencyLevel;\nimport com.datastax.driver.core.BoundStatement;\nimport com.datastax.driver.core.ResultSetFuture;\nimport com.datastax.driver.core.SSLOptions;\nimport com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions;\nimport com.datastax.driver.core.policies.TokenAwarePolicy;\nimport com.datastax.driver.core.policies.DCAwareRoundRobinPolicy;\n\nimport com.codahale.metrics.Timer;\nimport org.apache.commons.lang3.StringEscapeUtils;\n\npublic class CqlDelimLoad {\n private String version = \"0.0.27\";\n private String host = null;\n private int port = 9042;\n private String username = null;\n private String password = null;\n private String truststorePath = null;\n private String truststorePwd = null;\n private String keystorePath = null;\n private String keystorePwd = null;\n private Cluster cluster = null;\n private Session session = null;\n private ConsistencyLevel consistencyLevel = ConsistencyLevel.LOCAL_ONE;\n private int numFutures = 1000;\n private int inNumFutures = -1;\n private int queryTimeout = 2;\n private long maxInsertErrors = 10;\n private int numRetries = 1;\n private double rate = 50000.0;\n private long progressRate = 100000;\n private RateLimiter rateLimiter = null;\n private String rateFile = null;\n private PrintStream rateStream = null;\n private Integer inTtl = null;\n private int ttl = -1;\n\n private String cqlSchema = null;\n private String table = null;\n private String keyspace = null;\n private String format = \"delim\";\n\n private long maxErrors = 10;\n private long skipRows = 0;\n private String skipCols = null;\n \n private long maxRows = -1;\n private String badDir = \".\";\n private String filename = null;\n public static String STDIN = \"stdin\";\n public static String STDERR = \"stderr\";\n private String successDir = null;\n private String failureDir = null;\n\n private Locale locale = null;\n private BooleanParser.BoolStyle boolStyle = null;\n private String dateFormatString = null;\n private String localDateFormatString = \"yyyy-MM-dd\";\n private String nullString = null;\n private String commentString = null;\n private String delimiter = null;\n private int charsPerColumn = 4096;\n\n private int numThreads = Runtime.getRuntime().availableProcessors();\n private int batchSize = 1;\n private boolean nullsUnset = false;\n\n private String usage() {\n StringBuilder usage = new StringBuilder(\"version: \").append(version).append(\"\\n\");\n usage.append(\"Usage: -f -host [OPTIONS]\\n\");\n usage.append(\"OPTIONS:\\n\");\n usage.append(\" -schema Table schema (when using delim)\\n\");\n usage.append(\" -table Table name (when using json)\\n\");\n usage.append(\" -keyspace Keyspace name (when using json)\\n\");\n usage.append(\" -configFile File with configuration options\\n\");\n usage.append(\" -delim Delimiter to use [,]\\n\");\n usage.append(\" -charsPerColumn Max number of chars per column [4096]\\n\");\n usage.append(\" -dateFormat Date format for TIMESTAMP [default for Locale.ENGLISH]\\n\");\n usage.append(\" -localDateFormat Date format for DATE [yyyy-MM-dd]\\n\");\n usage.append(\" -nullString String that signifies NULL [none]\\n\");\n usage.append(\" -comment Comment symbol to use [none]\\n\");\n usage.append(\" -skipRows Number of rows to skip [0]\\n\");\n usage.append(\" -skipCols Comma-separated list of columsn to skip in the input file\\n\");\n usage.append(\" -maxRows Maximum number of rows to read (-1 means all) [-1]\\n\");\n usage.append(\" -maxErrors Maximum parse errors to endure [10]\\n\");\n usage.append(\" -badDir Directory for where to place badly parsed rows. [none]\\n\");\n usage.append(\" -port CQL Port Number [9042]\\n\");\n usage.append(\" -user Cassandra username [none]\\n\");\n usage.append(\" -pw Password for user [none]\\n\");\n usage.append(\" -ssl-truststore-path Path to SSL truststore [none]\\n\");\n usage.append(\" -ssl-truststore-pw Password for SSL truststore [none]\\n\");\n usage.append(\" -ssl-keystore-path Path to SSL keystore [none]\\n\");\n usage.append(\" -ssl-keystore-pw Password for SSL keystore [none]\\n\");\n usage.append(\" -consistencyLevel Consistency level [LOCAL_ONE]\\n\");\n usage.append(\" -numFutures Number of CQL futures to keep in flight [1000]\\n\");\n usage.append(\" -batchSize Number of INSERTs to batch together [1]\\n\");\n usage.append(\" -decimalDelim Decimal delimiter [.] Other option is ','\\n\");\n usage.append(\" -boolStyle Style for booleans [TRUE_FALSE]\\n\");\n usage.append(\" -numThreads Number of concurrent threads (files) to load [num cores]\\n\");\n usage.append(\" -queryTimeout <# seconds> Query timeout (in seconds) [2]\\n\");\n usage.append(\" -numRetries Number of times to retry the INSERT [1]\\n\");\n usage.append(\" -maxInsertErrors <# errors> Maximum INSERT errors to endure [10]\\n\");\n usage.append(\" -rate Maximum insert rate [50000]\\n\");\n usage.append(\" -progressRate How often to report the insert rate [100000]\\n\");\n usage.append(\" -rateFile Where to print the rate statistics\\n\");\n usage.append(\" -successDir Directory where to move successfully loaded files\\n\");\n usage.append(\" -failureDir Directory where to move files that did not successfully load\\n\");\n usage.append(\" -nullsUnset [false|true] Treat nulls as unset [faslse]\\n\");\n usage.append(\" -format [delim|jsonline|jsonarray] Format of data: delimited or JSON [delim]\\n\");\n usage.append(\" -table Table name (when using JSON)\\n\");\n usage.append(\" -keyspace Keyspace name (when using JSON)\\n\");\n usage.append(\" -ttl TTL for all rows in this invocation [unset]\\n\");\n\n usage.append(\"\\n\\nExamples:\\n\");\n usage.append(\"cassandra-loader -f /path/to/file.csv -host localhost -schema \\\"test.test3(a, b, c)\\\"\\n\");\n usage.append(\"cassandra-loader -f /path/to/directory -host 1.2.3.4 -schema \\\"test.test3(a, b, c)\\\" -delim \\\"\\\\t\\\" -numThreads 10\\n\");\n usage.append(\"cassandra-loader -f stdin -host localhost -schema \\\"test.test3(a, b, c)\\\" -user myuser -pw mypassword\\n\");\n return usage.toString();\n }\n \n private boolean validateArgs() {\n if (format.equalsIgnoreCase(\"delim\")) {\n if (null == cqlSchema) {\n System.err.println(\"If you specify format \" + format + \" you must provide a schema\");\n return false;\n }\n if (null != keyspace)\n System.err.println(\"Format is \" + format + \", ignoring keyspace\");\n if (null != table)\n System.err.println(\"Format is \" + format + \", ignoring table\");\n }\n else if (format.equalsIgnoreCase(\"jsonline\") \n || format.equalsIgnoreCase(\"jsonarray\")) {\n if (null == keyspace) {\n System.err.println(\"If you specify format \" + format + \" you must provide a keyspace\");\n return false;\n }\n if (null == table) {\n System.err.println(\"If you specify format \" + format + \" you must provide a table\");\n return false;\n }\n if (null != cqlSchema)\n System.err.println(\"Format is \" + format + \", ignoring schema\");\n }\n else {\n System.err.println(\"Unknown format option\");\n return false;\n }\n\n if (0 >= numFutures) {\n System.err.println(\"Number of futures must be positive (\" + numFutures + \")\");\n return false;\n }\n if (0 >= batchSize) {\n System.err.println(\"Batch size must be positive (\" + batchSize + \")\");\n return false;\n }\n if (0 >= queryTimeout) {\n System.err.println(\"Query timeout must be positive\");\n return false;\n }\n if (0 > maxInsertErrors) {\n System.err.println(\"Maximum number of insert errors must be non-negative\");\n return false;\n }\n if (0 > numRetries) {\n System.err.println(\"Number of retries must be non-negative\");\n return false;\n }\n if (0 > skipRows) {\n System.err.println(\"Number of rows to skip must be non-negative\");\n return false;\n }\n if (0 >= maxRows) {\n System.err.println(\"Maximum number of rows to load must be positive\");\n return false;\n }\n if (0 > maxErrors) {\n System.err.println(\"Maximum number of parse errors must be non-negative\");\n return false;\n }\n if (0 > progressRate) {\n System.err.println(\"Progress rate must be non-negative\");\n return false;\n }\n if (1 > numThreads) {\n System.err.println(\"Number of threads must be non-negative\");\n return false;\n }\n if (!STDIN.equalsIgnoreCase(filename)) {\n File infile = new File(filename);\n if ((!infile.isFile()) && (!infile.isDirectory())) {\n System.err.println(\"The -f argument needs to be a file or a directory\");\n return false;\n }\n if (infile.isDirectory()) {\n File[] infileList = infile.listFiles();\n if (infileList.length < 1) {\n System.err.println(\"The directory supplied is empty\");\n return false;\n }\n }\n }\n if (null != successDir) {\n if (STDIN.equalsIgnoreCase(filename)) {\n System.err.println(\"Cannot specify -successDir with stdin\");\n return false;\n }\n File sdir = new File(successDir);\n if (!sdir.isDirectory()) {\n System.err.println(\"-successDir must be a directory\");\n return false;\n }\n }\n if (null != failureDir) {\n if (STDIN.equalsIgnoreCase(filename)) {\n System.err.println(\"Cannot specify -failureDir with stdin\");\n return false;\n }\n File sdir = new File(failureDir);\n if (!sdir.isDirectory()) {\n System.err.println(\"-failureDir must be a directory\");\n return false;\n }\n }\n if ((null == username) && (null != password)) {\n System.err.println(\"If you supply the password, you must supply the username\");\n return false;\n }\n if ((null != username) && (null == password)) {\n System.err.println(\"If you supply the username, you must supply the password\");\n return false;\n }\n if ((null == truststorePath) && (null != truststorePwd)) {\n System.err.println(\"If you supply the ssl-truststore-pw, you must supply the ssl-truststore-path\");\n return false;\n }\n if ((null != truststorePath) && (null == truststorePwd)) {\n System.err.println(\"If you supply the ssl-truststore-path, you must supply the ssl-truststore-pw\");\n return false;\n }\n if ((null == keystorePath) && (null != keystorePwd)) {\n System.err.println(\"If you supply the ssl-keystore-pw, you must supply the ssl-keystore-path\");\n return false;\n }\n if ((null != keystorePath) && (null == keystorePwd)) {\n System.err.println(\"If you supply the ssl-keystore-path, you must supply the ssl-keystore-pw\");\n return false;\n }\n File tfile = null;\n if (null != truststorePath) {\n tfile = new File(truststorePath);\n if (!tfile.isFile()) {\n System.err.println(\"truststore file must be a file\");\n return false;\n }\n }\n if (null != keystorePath) {\n tfile = new File(keystorePath);\n if (!tfile.isFile()) {\n System.err.println(\"keystore file must be a file\");\n return false;\n }\n }\n\n if (0 > rate) {\n System.err.println(\"Rate must be positive\");\n return false;\n }\n if (0 > charsPerColumn) {\n System.err.println(\"charsPerColumn must be positive\");\n return false;\n }\n\n return true;\n }\n\n private boolean processConfigFile(String fname, Map amap)\n throws IOException, FileNotFoundException {\n File cFile = new File(fname);\n if (!cFile.isFile()) {\n System.err.println(\"Configuration File must be a file\");\n return false;\n }\n\n BufferedReader cReader = new BufferedReader(new FileReader(cFile));\n String line;\n while ((line = cReader.readLine()) != null) {\n String[] fields = line.trim().split(\"\\\\s+\");\n if (2 != fields.length) {\n System.err.println(\"Bad line in config file: \" + line);\n return false;\n }\n if (null == amap.get(fields[0])) {\n amap.put(fields[0], fields[1]);\n }\n }\n return true;\n }\n \n private boolean parseArgs(String[] args) throws IOException, FileNotFoundException {\n String tkey;\n if (args.length == 0) {\n System.err.println(\"No arguments specified\");\n return false;\n }\n if (0 != args.length % 2) {\n System.err.println(\"Not an even number of parameters\");\n return false;\n }\n\n Map amap = new HashMap();\n for (int i = 0; i < args.length; i+=2)\n amap.put(args[i], args[i+1]);\n\n if (null != (tkey = amap.remove(\"-configFile\")))\n if (!processConfigFile(tkey, amap))\n return false;\n\n host = amap.remove(\"-host\");\n if (null == host) { // host is required\n System.err.println(\"Must provide a host\");\n return false;\n }\n\n filename = amap.remove(\"-f\");\n if (null == filename) { // filename is required\n System.err.println(\"Must provide a filename/directory\");\n return false;\n }\n\n if (null != (tkey = amap.remove(\"-format\"))) format = tkey;\n if (null != (tkey = amap.remove(\"-schema\"))) cqlSchema = tkey;\n if (null != (tkey = amap.remove(\"-table\"))) table = tkey;\n if (null != (tkey = amap.remove(\"-keyspace\"))) keyspace = tkey;\n\n if (null != (tkey = amap.remove(\"-port\"))) port = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-user\"))) username = tkey;\n if (null != (tkey = amap.remove(\"-pw\"))) password = tkey;\n if (null != (tkey = amap.remove(\"-ssl-truststore-path\"))) truststorePath = tkey;\n if (null != (tkey = amap.remove(\"-ssl-truststore-pw\"))) truststorePwd = tkey;\n if (null != (tkey = amap.remove(\"-ssl-keystore-path\"))) keystorePath = tkey;\n if (null != (tkey = amap.remove(\"-ssl-keystore-pw\"))) keystorePwd = tkey;\n if (null != (tkey = amap.remove(\"-consistencyLevel\"))) consistencyLevel = ConsistencyLevel.valueOf(tkey);\n if (null != (tkey = amap.remove(\"-numFutures\"))) inNumFutures = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-batchSize\"))) batchSize = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-queryTimeout\"))) queryTimeout = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-maxInsertErrors\"))) maxInsertErrors = Long.parseLong(tkey);\n if (null != (tkey = amap.remove(\"-numRetries\"))) numRetries = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-maxErrors\"))) maxErrors = Long.parseLong(tkey);\n if (null != (tkey = amap.remove(\"-skipRows\"))) skipRows = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-skipCols\"))) skipCols = tkey;\n if (null != (tkey = amap.remove(\"-maxRows\"))) maxRows = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-badDir\"))) badDir = tkey;\n if (null != (tkey = amap.remove(\"-dateFormat\"))) dateFormatString = tkey;\n if (null != (tkey = amap.remove(\"-localDateFormat\"))) localDateFormatString = tkey;\n if (null != (tkey = amap.remove(\"-nullString\"))) nullString = tkey;\n if (null != (tkey = amap.remove(\"-comment\"))) commentString = tkey;\n if (null != (tkey = amap.remove(\"-delim\"))) delimiter = tkey;\n if (null != (tkey = amap.remove(\"-numThreads\"))) numThreads = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-rate\"))) rate = Double.parseDouble(tkey);\n if (null != (tkey = amap.remove(\"-progressRate\"))) progressRate = Long.parseLong(tkey);\n if (null != (tkey = amap.remove(\"-ttl\"))) inTtl = new Integer(tkey);\n if (null != (tkey = amap.remove(\"-rateFile\"))) rateFile = tkey;\n if (null != (tkey = amap.remove(\"-successDir\"))) successDir = tkey;\n if (null != (tkey = amap.remove(\"-failureDir\"))) failureDir = tkey;\n if (null != (tkey = amap.remove(\"-decimalDelim\"))) {\n if (tkey.equals(\",\"))\n locale = Locale.FRANCE;\n }\n if (null != (tkey = amap.remove(\"-boolStyle\"))) {\n boolStyle = BooleanParser.getBoolStyle(tkey);\n if (null == boolStyle) {\n System.err.println(\"Bad boolean style. Options are: \" + BooleanParser.getOptions());\n return false;\n }\n }\n if (null != (tkey = amap.remove(\"-nullsUnset\"))) nullsUnset = Boolean.parseBoolean(tkey);\n if (null != (tkey = amap.remove(\"-charsPerColumn\"))) charsPerColumn = Integer.parseInt(tkey);\n\n if (-1 == maxRows)\n maxRows = Long.MAX_VALUE;\n if (-1 == maxErrors)\n maxErrors = Long.MAX_VALUE;\n if (-1 == maxInsertErrors)\n maxInsertErrors = Long.MAX_VALUE;\n \n if (!amap.isEmpty()) {\n for (String k : amap.keySet())\n System.err.println(\"Unrecognized option: \" + k);\n return false;\n }\n\n if (0 < inNumFutures)\n numFutures = inNumFutures / numThreads;\n\tif (null != inTtl) {\n\t if (1 > inTtl.intValue()) {\n\t\tSystem.err.println(\"TTL must be greater than 1\");\n\t\treturn false;\n\t }\n\t else\n\t\tttl = inTtl.intValue();\n\t}\n\n if (null != keyspace)\n keyspace = quote(keyspace);\n if (null != table)\n table = quote(table);\n\n return validateArgs();\n }\n\n private String quote(String instr) {\n String ret = instr;\n if ((ret.startsWith(\"\\\"\")) && (ret.endsWith(\"\\\"\")))\n ret = ret.substring(1, ret.length() - 1);\n return \"\\\"\" + ret + \"\\\"\";\n }\n\n private SSLOptions createSSLOptions() \n throws KeyStoreException, FileNotFoundException, IOException, NoSuchAlgorithmException, \n KeyManagementException, CertificateException, UnrecoverableKeyException {\n TrustManagerFactory tmf = null;\n KeyStore tks = KeyStore.getInstance(\"JKS\");\n tks.load((InputStream) new FileInputStream(new File(truststorePath)), \n truststorePwd.toCharArray());\n tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());\n tmf.init(tks);\n \n KeyManagerFactory kmf = null;\n if (null != keystorePath) {\n KeyStore kks = KeyStore.getInstance(\"JKS\");\n kks.load((InputStream) new FileInputStream(new File(keystorePath)), \n keystorePwd.toCharArray());\n kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());\n kmf.init(kks, keystorePwd.toCharArray());\n }\n\n SSLContext sslContext = SSLContext.getInstance(\"TLS\");\n sslContext.init(kmf != null? kmf.getKeyManagers() : null, \n tmf != null ? tmf.getTrustManagers() : null, \n new SecureRandom());\n\n return RemoteEndpointAwareJdkSSLOptions.builder().withSSLContext(sslContext).build();\n }\n\n private boolean setup() \n throws IOException, KeyStoreException, NoSuchAlgorithmException, KeyManagementException,\n CertificateException, UnrecoverableKeyException {\n // Connect to Cassandra\n Session tsession = null;\n try {\n PoolingOptions pOpts = new PoolingOptions();\n pOpts.setMaxConnectionsPerHost(HostDistance.LOCAL, 8);\n pOpts.setCoreConnectionsPerHost(HostDistance.LOCAL, 8);\n Cluster.Builder clusterBuilder = Cluster.builder()\n .addContactPoint(host)\n .withPort(port)\n //.withCompression(ProtocolOptions.Compression.LZ4)\n .withPoolingOptions(pOpts)\n .withLoadBalancingPolicy(new TokenAwarePolicy( DCAwareRoundRobinPolicy.builder().build()))\n ;\n\n if (null != username)\n clusterBuilder = clusterBuilder.withCredentials(username, password);\n if (null != truststorePath)\n clusterBuilder = clusterBuilder.withSSL(createSSLOptions());\n\n cluster = clusterBuilder.build();\n if (null == cluster) {\n throw new IOException(\"Could not create cluster\");\n }\n tsession = cluster.connect();\n }\n catch (IllegalArgumentException e){\n System.err.println(\"Could not connect to the cluster, check your hosts\");\n //e.printStackTrace();\n return false;\n }\n catch (Exception e){\n System.err.println(e.getStackTrace());\n e.printStackTrace();\n return false;\n }\n\n if ((0 > cluster.getConfiguration().getProtocolOptions()\n .getProtocolVersion().compareTo(ProtocolVersion.V4))\n && nullsUnset) {\n System.err.println(\"Cannot use nullsUnset with ProtocolVersion less than V4 (prior to Cassandra 3.0\");\n cleanup();\n return false;\n }\n\n if (null != rateFile) {\n if (STDERR.equalsIgnoreCase(rateFile)) {\n rateStream = System.err;\n }\n else {\n rateStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(rateFile)), true);\n }\n }\n Metrics metrics = cluster.getMetrics();\n com.codahale.metrics.Timer timer = metrics.getRequestsTimer();\n rateLimiter = new RateLimiter(rate, progressRate, timer, rateStream);\n //rateLimiter = new Latency999RateLimiter(rate, progressRate, 3000, 200, 10, 0.5, 0.1, cluster, false);\n session = new RateLimitedSession(tsession, rateLimiter);\n\n return true;\n }\n\n private void cleanup() {\n rateLimiter.report(null, null);\n if (null != rateStream)\n rateStream.close();\n if (null != session)\n session.close();\n if (null != cluster)\n cluster.close();\n }\n \n public boolean run(String[] args) \n throws IOException, ParseException, InterruptedException, ExecutionException, KeyStoreException,\n NoSuchAlgorithmException, KeyManagementException, CertificateException, \n UnrecoverableKeyException {\n if (false == parseArgs(args)) {\n System.err.println(\"Bad arguments\");\n System.err.println(usage());\n return false;\n }\n\n // Setup\n if (false == setup())\n return false;\n \n // open file\n Deque fileList = new ArrayDeque();\n File infile = null;\n File[] inFileList = null;\n boolean onefile = true;\n if (STDIN.equalsIgnoreCase(filename)) {\n infile = null;\n }\n else {\n infile = new File(filename);\n if (infile.isFile()) {\n }\n else {\n inFileList = infile.listFiles();\n if (inFileList.length < 1)\n throw new IOException(\"directory is empty\");\n onefile = false;\n Arrays.sort(inFileList, \n new Comparator() {\n public int compare(File f1, File f2) {\n return f1.getName().compareTo(f2.getName());\n }\n });\n for (int i = 0; i < inFileList.length; i++)\n fileList.push(inFileList[i]);\n }\n }\n\n // Launch Threads\n ExecutorService executor;\n long total = 0;\n if (onefile) {\n // One file/stdin to process\n executor = Executors.newSingleThreadExecutor();\n Callable worker = new CqlDelimLoadTask(cqlSchema, delimiter, \n charsPerColumn, nullString,\n commentString,\n dateFormatString, \n localDateFormatString, \n boolStyle, locale, \n maxErrors, skipRows,\n skipCols,\n maxRows, badDir, infile, \n session, consistencyLevel,\n numFutures, batchSize,\n numRetries, queryTimeout,\n maxInsertErrors, \n successDir, failureDir,\n nullsUnset, format,\n keyspace, table, ttl);\n Future res = executor.submit(worker);\n total = res.get();\n executor.shutdown();\n }\n else {\n executor = Executors.newFixedThreadPool(numThreads);\n Set> results = new HashSet>();\n while (!fileList.isEmpty()) {\n File tFile = fileList.pop();\n Callable worker = new CqlDelimLoadTask(cqlSchema, delimiter,\n charsPerColumn, nullString,\n commentString,\n dateFormatString, \n localDateFormatString, \n boolStyle, locale, \n maxErrors, skipRows,\n skipCols,\n maxRows, badDir, tFile, \n session,\n consistencyLevel,\n numFutures, batchSize,\n numRetries, \n queryTimeout,\n maxInsertErrors, \n successDir, failureDir,\n nullsUnset, format,\n keyspace, table, ttl);\n results.add(executor.submit(worker));\n }\n executor.shutdown();\n for (Future res : results)\n total += res.get();\n }\n\n // Cleanup\n cleanup();\n //System.err.println(\"Total rows inserted: \" + total);\n\n return true;\n }\n\n public static void main(String[] args) \n throws IOException, ParseException, InterruptedException, ExecutionException, \n KeyStoreException, NoSuchAlgorithmException, UnrecoverableKeyException, \n CertificateException, KeyManagementException {\n CqlDelimLoad cdl = new CqlDelimLoad();\n boolean success = cdl.run(args);\n if (success) {\n System.exit(0);\n } else {\n System.exit(-1);\n }\n }\n}\n\n" }, { "path": "src/main/java/com/datastax/loader/CqlDelimLoadTask.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport com.datastax.driver.core.BatchStatement;\nimport com.datastax.driver.core.BoundStatement;\nimport com.datastax.driver.core.ConsistencyLevel;\nimport com.datastax.driver.core.PreparedStatement;\nimport com.datastax.driver.core.ResultSetFuture;\nimport com.datastax.driver.core.Session;\nimport com.datastax.loader.futures.FutureManager;\nimport com.datastax.loader.futures.PrintingFutureSet;\nimport com.datastax.loader.futures.JsonPrintingFutureSet;\nimport com.datastax.loader.parser.BooleanParser;\nimport org.json.simple.JSONArray;\nimport org.json.simple.JSONObject;\nimport org.json.simple.parser.JSONParser;\n\nimport java.io.BufferedOutputStream;\nimport java.io.BufferedReader;\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.FileInputStream;\nimport java.io.InputStream;\nimport java.io.IOException;\nimport java.io.InputStreamReader;\nimport java.io.PrintStream;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.text.ParseException;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.TimeUnit;\nimport java.util.zip.GZIPInputStream;\nimport java.util.zip.ZipException;\n\nclass CqlDelimLoadTask implements Callable {\n private String BADPARSE = \".BADPARSE\";\n private String BADINSERT = \".BADINSERT\";\n private String LOG = \".LOG\";\n private Session session;\n private String insert;\n private PreparedStatement statement;\n private BatchStatement batch;\n private StringBuilder batchString;\n private FutureManager fm;\n private ConsistencyLevel consistencyLevel;\n private CqlDelimParser cdp;\n private long maxErrors;\n private long skipRows;\n private String skipCols = null;\n private long maxRows;\n private String badDir;\n private String successDir;\n private String failureDir;\n private String readerName;\n private PrintStream badParsePrinter = null;\n private PrintStream badInsertPrinter = null;\n private PrintStream logPrinter = null;\n private String logFname = \"\";\n private BufferedReader reader;\n private File infile;\n private int numFutures;\n private int batchSize;\n private long numInserted;\n private int ttl = -1;\n\n private String cqlSchema;\n private Locale locale = null;\n private BooleanParser.BoolStyle boolStyle = null;\n private String dateFormatString = null;\n private String localDateFormatString = null;\n private String nullString = null;\n private String commentString = null;\n private String delimiter = null;\n private int charsPerColumn = 4096;\n private TimeUnit unit = TimeUnit.SECONDS;\n private long queryTimeout = 2;\n private int numRetries = 1;\n private long maxInsertErrors = 10;\n private long insertErrors = 0;\n private boolean nullsUnset;\n private String format = \"delim\";\n private String keyspace = null;\n private String table = null;\n private JSONArray jsonArray;\n\n public CqlDelimLoadTask(String inCqlSchema, String inDelimiter, \n int inCharsPerColumn,\n String inNullString, String inCommentString, \n String inDateFormatString, String inLocalDateFormatString,\n BooleanParser.BoolStyle inBoolStyle, \n Locale inLocale, \n long inMaxErrors, long inSkipRows, \n String inSkipCols, long inMaxRows,\n String inBadDir, File inFile,\n Session inSession, ConsistencyLevel inCl,\n int inNumFutures, int inBatchSize, int inNumRetries, \n int inQueryTimeout, long inMaxInsertErrors,\n String inSuccessDir, String inFailureDir,\n boolean inNullsUnset, String inFormat,\n String inKeyspace, String inTable, int inTtl) {\n super();\n cqlSchema = inCqlSchema;\n delimiter = inDelimiter;\n charsPerColumn = inCharsPerColumn;\n nullString = inNullString;\n commentString = inCommentString;\n dateFormatString = inDateFormatString;\n localDateFormatString = inLocalDateFormatString;\n boolStyle = inBoolStyle;\n locale = inLocale;\n maxErrors = inMaxErrors;\n skipRows = inSkipRows;\n skipCols = inSkipCols;\n maxRows = inMaxRows;\n badDir = inBadDir;\n infile = inFile;\n session = inSession;\n consistencyLevel = inCl;\n numFutures = inNumFutures;\n batchSize = inBatchSize;\n numRetries = inNumRetries;\n queryTimeout = inQueryTimeout;\n maxInsertErrors = inMaxInsertErrors;\n successDir = inSuccessDir;\n failureDir = inFailureDir;\n nullsUnset = inNullsUnset;\n format = inFormat;\n keyspace = inKeyspace;\n table = inTable;\n\tttl = inTtl;\n }\n\n public Long call() throws IOException, ParseException, org.json.simple.parser.ParseException {\n setup();\n numInserted = execute();\n return numInserted;\n }\n\n private void setup() throws IOException, ParseException, org.json.simple.parser.ParseException {\n if (null == infile) {\n reader = new BufferedReader(new InputStreamReader(System.in));\n readerName = \"stdin\";\n }\n else {\n InputStream is = null;\n try {\n is = new GZIPInputStream(new FileInputStream(infile));\n }\n catch (ZipException e) {\n is = new FileInputStream(infile);\n }\n reader = new BufferedReader(new InputStreamReader(is));\n readerName = infile.getName();\n }\n\n //setup json reader\n if(format.equalsIgnoreCase(\"jsonarray\")){\n JSONParser parser = new JSONParser();\n jsonArray= (JSONArray) parser.parse(reader);\n }\n\n // Prepare Badfile\n if (null != badDir) {\n badParsePrinter = new PrintStream(new BufferedOutputStream(new FileOutputStream(badDir + \"/\" + readerName + BADPARSE)));\n badInsertPrinter = new PrintStream(new BufferedOutputStream(new FileOutputStream(badDir + \"/\" + readerName + BADINSERT)));\n logFname = badDir + \"/\" + readerName + LOG;\n logPrinter = new PrintStream(new BufferedOutputStream(new FileOutputStream(logFname)));\n }\n\n if (format.equalsIgnoreCase(\"delim\")) {\n cdp = new CqlDelimParser(cqlSchema, delimiter, charsPerColumn, \n nullString, commentString,\n dateFormatString, localDateFormatString,\n boolStyle, locale,\n skipCols, session, true, ttl);\n }\n else if (format.equalsIgnoreCase(\"jsonline\")\n || format.equalsIgnoreCase(\"jsonarray\")) {\n cdp = new CqlDelimParser(keyspace, table, delimiter, charsPerColumn,\n nullString, commentString,\n dateFormatString, localDateFormatString,\n boolStyle, locale, \n skipCols, session, true, ttl);\n }\n\n insert = cdp.generateInsert();\n statement = session.prepare(insert);\n statement.setRetryPolicy(new LoaderRetryPolicy(numRetries));\n statement.setConsistencyLevel(consistencyLevel);\n batch = new BatchStatement(BatchStatement.Type.UNLOGGED);\n batchString = new StringBuilder();\n if (format.equalsIgnoreCase(\"delim\")) {\n fm = new PrintingFutureSet(numFutures, queryTimeout, \n maxInsertErrors, logPrinter, \n badInsertPrinter);\n }\n else if (format.equalsIgnoreCase(\"jsonline\")\n || format.equalsIgnoreCase(\"jsonarray\")) {\n fm = new JsonPrintingFutureSet(numFutures, queryTimeout, \n maxInsertErrors, logPrinter, \n badInsertPrinter);\n }\n }\n \n private void cleanup(boolean success) throws IOException {\n if (null != badParsePrinter) {\n if (format.equalsIgnoreCase(\"jsonarray\"))\n badParsePrinter.println(\"]\");\n badParsePrinter.close();\n }\n if (null != badInsertPrinter) {\n if (format.equalsIgnoreCase(\"jsonarray\"))\n badInsertPrinter.println(\"]\");\n badInsertPrinter.close();\n }\n if (null != logPrinter)\n logPrinter.close();\n if (success) {\n if (null != successDir) {\n Path src = infile.toPath();\n Path dst = Paths.get(successDir);\n Files.move(src, dst.resolve(src.getFileName()), \n StandardCopyOption.REPLACE_EXISTING);\n }\n }\n else {\n if (null != failureDir) {\n Path src = infile.toPath();\n Path dst = Paths.get(failureDir);\n Files.move(src, dst.resolve(src.getFileName()), \n StandardCopyOption.REPLACE_EXISTING);\n }\n }\n }\n\n private int sendInsert(List elements, String line) {\n BoundStatement bind = statement.bind(elements.toArray());\n ResultSetFuture resultSetFuture;\n int retval = 0;\n if (nullsUnset) {\n for (int i = 0; i < elements.size(); i++)\n if (null == elements.get(i))\n bind.unset(i);\n }\n if (1 == batchSize) {\n resultSetFuture = session.executeAsync(bind);\n if (!fm.add(resultSetFuture, line)) {\n System.err.println(\"There was an error. Please check the log file for more information (\" + logFname + \")\");\n //cleanup(false);\n return -2;\n }\n //numInserted += 1;\n retval = 1;\n }\n else {\n batch.add(bind);\n batchString.append(\"\\n\").append(line);\n if (batchSize == batch.size()) {\n resultSetFuture = session.executeAsync(batch);\n if (!fm.add(resultSetFuture, batchString.toString())) {\n System.err.println(\"There was an error. Please check the log file for more information (\" + logFname + \")\");\n //cleanup(false);\n return -2;\n }\n int numInserted = batch.size();\n batch.clear();\n batchString.setLength(0);\n retval = numInserted;\n }\n }\n return retval;\n }\n\n private long execute() throws IOException {\n String line = null;\n int lineNumber = 0;\n long numInserted = 0;\n int numErrors = 0;\n int curBatch = 0;\n BoundStatement bind = null;\n List elements = null;\n\n System.err.println(\"*** Processing \" + readerName);\n if (format.equalsIgnoreCase(\"delim\")\n || format.equalsIgnoreCase(\"jsonline\")) {\n while ((line = reader.readLine()) != null) {\n lineNumber++;\n if (skipRows > 0) {\n skipRows--;\n continue;\n }\n if (maxRows-- < 0)\n break;\n\n if (0 == line.trim().length())\n continue;\n\n elements = null;\n if (format.equalsIgnoreCase(\"delim\"))\n elements = cdp.parse(line);\n else if (format.equalsIgnoreCase(\"jsonline\"))\n elements = cdp.parseJson(line);\n if (null != elements) {\n int ret = sendInsert(elements, line);\n if (-2 == ret) {\n cleanup(false);\n return -2;\n }\n numInserted += ret;\n }\n else {\n if (null != logPrinter) {\n logPrinter.println(String.format(\"Error parsing line %d in %s: %s\", lineNumber, readerName, line));\n }\n System.err.println(String.format(\"Error parsing line %d in %s: %s\", lineNumber, readerName, line));\n if (null != badParsePrinter) {\n badParsePrinter.println(line);\n }\n numErrors++;\n if (maxErrors <= numErrors) {\n if (null != logPrinter) {\n logPrinter.println(String.format(\"Maximum number of errors exceeded (%d) for %s\", numErrors, readerName));\n }\n System.err.println(String.format(\"Maximum number of errors exceeded (%d) for %s\", numErrors, readerName));\n cleanup(false);\n return -1;\n }\n }\n }\n } // if (format.equalsIgnoreCase(\"delim\"))\n else if (format.equalsIgnoreCase(\"jsonarray\")) {\n boolean firstBadJson = true;\n String badJsonDelim = \"[\\n\";\n List columnBackbone = cdp.getColumnNames();\n int columnCount = columnBackbone.size();\n for (Object o : jsonArray) {\n JSONObject jsonRow = (JSONObject) o;\n String[] jsonElements = new String[columnCount];\n jsonElements[0] = jsonRow.get(columnBackbone.get(0)).toString();\n for (int i = 1; i < columnCount; i++) {\n if (null != jsonRow.get(columnBackbone.get(i))) {\n jsonElements[i] = jsonRow.get(columnBackbone.get(i)).toString();\n } else {\n jsonElements[i] = null;\n }\n }\n if (null != (elements = cdp.parse(jsonElements))) {\n int ret = sendInsert(elements, line);\n if (-2 == ret) {\n cleanup(false);\n return -2;\n }\n numInserted += ret;\n } else {\n String badString = jsonRow.toJSONString();\n if (null != logPrinter) {\n logPrinter.println(String.format(\"Error parsing JSON item %d in %s: %s\", lineNumber, readerName, badString));\n }\n System.err.println(String.format(\"Error parsing JSON item %d in %s: %s\", lineNumber, readerName, badString));\n if (null != badParsePrinter) {\n badParsePrinter.println(badJsonDelim + badString);\n if (firstBadJson) {\n firstBadJson = false;\n badJsonDelim = \",\\n\";\n }\n }\n numErrors++;\n if (maxErrors <= numErrors) {\n if (null != logPrinter) {\n logPrinter.println(String.format(\"Maximum number of errors exceeded (%d) for %s\", numErrors, readerName));\n }\n System.err.println(String.format(\"Maximum number of errors exceeded (%d) for %s\", numErrors, readerName));\n cleanup(false);\n return -1;\n }\n }\n }\n }// if (format.equalsIgnoreCase(\"json\"))\n\n // Send last partially filled batch\n if ((batchSize > 1) && (batch.size() > 0)) {\n ResultSetFuture resultSetFuture = session.executeAsync(batch);\n if (!fm.add(resultSetFuture, line)) {\n cleanup(false);\n return -2;\n }\n numInserted += batch.size();\n }\n\n if (!fm.cleanup()) {\n cleanup(false);\n return -1;\n }\n\n if (null != logPrinter) {\n logPrinter.println(\"*** DONE: \" + readerName + \" number of lines processed: \" + lineNumber + \" (\" + numInserted + \" inserted)\");\n }\n System.err.println(\"*** DONE: \" + readerName + \" number of lines processed: \" + lineNumber + \" (\" + numInserted + \" inserted)\");\n\n cleanup(true);\n return fm.getNumInserted();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/CqlDelimParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport com.datastax.driver.core.ColumnMetadata;\nimport com.datastax.driver.core.DataType;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.TableMetadata;\nimport com.datastax.driver.core.KeyspaceMetadata;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\nimport com.datastax.loader.parser.BigDecimalParser;\nimport com.datastax.loader.parser.BigIntegerParser;\nimport com.datastax.loader.parser.BooleanParser;\nimport com.datastax.loader.parser.ByteBufferParser;\nimport com.datastax.loader.parser.ByteParser;\nimport com.datastax.loader.parser.DateParser;\nimport com.datastax.loader.parser.DelimParser;\nimport com.datastax.loader.parser.DoubleParser;\nimport com.datastax.loader.parser.FloatParser;\nimport com.datastax.loader.parser.InetAddressParser;\nimport com.datastax.loader.parser.IntegerParser;\nimport com.datastax.loader.parser.ListParser;\nimport com.datastax.loader.parser.LocalDateParser;\nimport com.datastax.loader.parser.LongParser;\nimport com.datastax.loader.parser.MapParser;\nimport com.datastax.loader.parser.Parser;\nimport com.datastax.loader.parser.SetParser;\nimport com.datastax.loader.parser.ShortParser;\nimport com.datastax.loader.parser.StringParser;\nimport com.datastax.loader.parser.UUIDParser;\n\nimport java.text.ParseException;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport org.json.simple.JSONObject;\nimport org.json.simple.parser.JSONParser;\n\nimport com.univocity.parsers.csv.CsvParser;\nimport com.univocity.parsers.csv.CsvParserSettings;\n\npublic class CqlDelimParser {\n private Map pmap;\n private List sbl;\n private List columnNames;\n private String keyspace;\n private String tablename;\n private DelimParser delimParser;\n private JSONParser jsonParser;\n private int ttl = -1;\n\n public CqlDelimParser(String inCqlSchema, String inDelimiter, int inCharsPerColumn,\n String inNullString, String inCommentString, \n String inDateFormatString, String inLocalDateFormatString,\n BooleanParser.BoolStyle inBoolStyle, Locale inLocale,\n String skipList, Session session, boolean bLoader, int inTtl) \n throws ParseException {\n // Optionally provide things for the line parser - date format, boolean format, locale\n\tttl = inTtl;\n initPmap(inDateFormatString, inLocalDateFormatString, inBoolStyle, \n inLocale, bLoader);\n processCqlSchema(inCqlSchema, session);\n createDelimParser(inDelimiter, inCharsPerColumn, inNullString, inCommentString, skipList);\n } \n\n public CqlDelimParser(String inKeyspace, String inTable, String inDelimiter,\n int inCharsPerColumn,\n String inNullString, String inCommentString, \n String inDateFormatString, String inLocalDateFormatString,\n BooleanParser.BoolStyle inBoolStyle, Locale inLocale,\n String skipList, Session session, boolean bLoader, int inTtl) \n throws ParseException {\n // Optionally provide things for the line parser - date format, boolean format, locale\n\tttl = inTtl;\n keyspace = inKeyspace;\n tablename = inTable;\n initPmap(inDateFormatString, inLocalDateFormatString, inBoolStyle, \n inLocale, bLoader);\n processCqlSchema(session);\n createDelimParser(inDelimiter, inCharsPerColumn, inNullString, inCommentString, skipList);\n }\n\n public List getColumnNames() {\n return columnNames;\n }\n\n private void setColumnNames(List columnNames) {\n this.columnNames = columnNames;\n }\n\n // used internally to store schema information\n private class SchemaBits {\n public String name;\n public DataType.Name datatype;\n public Parser parser;\n }\n\n // intialize the Parsers and the parser map\n private void initPmap(String dateFormatString, String localDateFormatString,\n BooleanParser.BoolStyle inBoolStyle, \n Locale inLocale, boolean bLoader) {\n pmap = new HashMap();\n Parser byteParser = new ByteParser(inLocale, bLoader);\n Parser shortParser = new ShortParser(inLocale, bLoader);\n Parser integerParser = new IntegerParser(inLocale, bLoader);\n Parser longParser = new LongParser(inLocale, bLoader);\n Parser floatParser = new FloatParser(inLocale, bLoader);\n Parser doubleParser = new DoubleParser(inLocale, bLoader);\n Parser stringParser = new StringParser();\n Parser booleanParser = new BooleanParser(inBoolStyle);\n Parser uuidParser = new UUIDParser();\n Parser bigDecimalParser = new BigDecimalParser();\n Parser bigIntegerParser = new BigIntegerParser();\n Parser byteBufferParser = new ByteBufferParser();\n Parser inetAddressParser = new InetAddressParser();\n Parser dateParser = new DateParser(dateFormatString);\n Parser localDateParser = new LocalDateParser(localDateFormatString);\n\n pmap.put(DataType.Name.ASCII, stringParser);\n pmap.put(DataType.Name.BIGINT, longParser);\n pmap.put(DataType.Name.BLOB, byteBufferParser);\n pmap.put(DataType.Name.BOOLEAN, booleanParser);\n pmap.put(DataType.Name.COUNTER, longParser);\n pmap.put(DataType.Name.DATE , localDateParser);\n pmap.put(DataType.Name.DECIMAL, bigDecimalParser);\n pmap.put(DataType.Name.DOUBLE, doubleParser);\n pmap.put(DataType.Name.FLOAT, floatParser);\n pmap.put(DataType.Name.INET, inetAddressParser);\n pmap.put(DataType.Name.INT, integerParser);\n pmap.put(DataType.Name.SMALLINT , shortParser);\n pmap.put(DataType.Name.TEXT, stringParser);\n pmap.put(DataType.Name.TIME , longParser);\n pmap.put(DataType.Name.TIMESTAMP, dateParser);\n pmap.put(DataType.Name.TIMEUUID, uuidParser);\n pmap.put(DataType.Name.TINYINT , byteParser);\n pmap.put(DataType.Name.UUID, uuidParser);\n pmap.put(DataType.Name.VARCHAR, stringParser);\n pmap.put(DataType.Name.VARINT, bigIntegerParser);\n\n }\n\n // Validate the CQL schema, extract the keyspace and tablename, and process the rest of the schema\n private void processCqlSchema(String cqlSchema, Session session) throws ParseException {\n CsvParserSettings ks_settings = new CsvParserSettings();\n ks_settings.getFormat().setLineSeparator(\"\\n\");\n ks_settings.getFormat().setDelimiter('.');\n ks_settings.getFormat().setQuote('\\\"');\n ks_settings.getFormat().setQuoteEscape('\\\\');\n ks_settings.getFormat().setCharToEscapeQuoteEscaping('\\\\');\n ks_settings.setKeepQuotes(true);\n ks_settings.setKeepEscapeSequences(true);\n ks_settings.setMaxCharsPerColumn(-1);\n CsvParser ks_parser = new CsvParser(ks_settings);\n String[] ks_elements = ks_parser.parseLine(cqlSchema);\n keyspace = ks_elements[0];\n String table_string = cqlSchema.substring(keyspace.length() + 1);\n\n CsvParserSettings table_settings = new CsvParserSettings();\n table_settings.getFormat().setLineSeparator(\"\\n\");\n table_settings.getFormat().setDelimiter('(');\n table_settings.getFormat().setQuote('\\\"');\n table_settings.getFormat().setQuoteEscape('\\\\');\n table_settings.getFormat().setCharToEscapeQuoteEscaping('\\\\');\n table_settings.setKeepQuotes(true);\n table_settings.setKeepEscapeSequences(true);\n table_settings.setMaxCharsPerColumn(-1);\n CsvParser table_parser = new CsvParser(table_settings);\n String[] table_elements = table_parser.parseLine(table_string);\n tablename = table_elements[0];\n \n String schemaString = table_string.substring(tablename.length() + 1, \n table_string.length() - 1);\n\n sbl = schemaBits(schemaString, session);\n }\n\n private void processCqlSchema(Session session) throws ParseException {\n sbl = schemaBits(null, session);\n }\n\n\n private List schemaBits(String in, Session session) throws ParseException {\n KeyspaceMetadata km = session.getCluster().getMetadata().getKeyspace(keyspace);\n if (null == km) {\n System.err.println(\"Keyspace \" + keyspace + \" not found.\");\n System.exit(-1);\n }\n TableMetadata tm = km.getTable(tablename);\n if (null == tm) {\n System.err.println(\"Table \" + tablename + \" not found.\");\n System.exit(-1);\n }\n List inList = new ArrayList();\n if (null != in) {\n CsvParserSettings settings = new CsvParserSettings();\n settings.getFormat().setLineSeparator(\"\\n\");\n settings.getFormat().setDelimiter(',');\n settings.getFormat().setQuote('\\\"');\n settings.getFormat().setQuoteEscape('\\\\');\n settings.getFormat().setCharToEscapeQuoteEscaping('\\\\');\n settings.setKeepQuotes(true);\n settings.setKeepEscapeSequences(true);\n\t settings.setMaxCharsPerColumn(-1);\n CsvParser parser = new CsvParser(settings);\n String[] tlist = parser.parseLine(in);\n for (int i = 0; i < tlist.length; i++)\n inList.add(tlist[i].trim());\n }\n else {\n for (ColumnMetadata cm : tm.getColumns())\n inList.add(\"\\\"\"+cm.getName()+\"\\\"\");\n }\n //keep the list of columns from metadata to use as column backbone for JSON\n setColumnNames(inList);\n List sbl = new ArrayList();\n for (int i = 0; i < inList.size(); i++) {\n String col = inList.get(i);\n SchemaBits sb = new SchemaBits();\n ColumnMetadata cm = tm.getColumn(col);\n if (null == cm) {\n System.err.println(\"Column \" + col + \" of table \" + keyspace + \".\" + tablename + \" not found\");\n System.exit(-1);\n }\n DataType dt = cm.getType();\n sb.name = col;\n sb.datatype = dt.getName();\n if (dt.isCollection()) {\n if (sb.datatype == DataType.Name.LIST) {\n DataType.Name listType = dt.getTypeArguments().get(0).getName();\n Parser listParser = pmap.get(listType);\n if (null == listParser) {\n throw new ParseException(\"List data type not recognized (\" \n + listType + \")\", i);\n }\n sb.parser = new ListParser(listParser, ',', '[', ']');\n }\n else if (sb.datatype == DataType.Name.SET) {\n DataType.Name setType = dt.getTypeArguments().get(0).getName();\n Parser setParser = pmap.get(setType);\n if (null == setParser) {\n throw new ParseException(\"Set data type not recognized (\" \n + setType + \")\", i);\n }\n sb.parser = new SetParser(setParser, ',', '{', '}');\n }\n else if (sb.datatype == DataType.Name.MAP) {\n DataType.Name keyType = dt.getTypeArguments().get(0).getName();\n Parser keyParser = pmap.get(keyType);\n if (null == keyParser) {\n throw new ParseException(\"Map key data type not recognized (\" \n + keyType + \")\", i);\n }\n DataType.Name valueType = dt.getTypeArguments().get(1).getName();\n Parser valueParser = pmap.get(valueType);\n if (null == valueParser) {\n throw new ParseException(\"Map value data type not recognized (\" \n + valueType + \")\", i);\n }\n sb.parser = new MapParser(keyParser, valueParser, ',', '{', '}', ':');\n }\n else {\n throw new ParseException(\"Collection data type not recognized (\" \n + sb.datatype + \")\", i);\n }\n }\n else {\n sb.parser = pmap.get(sb.datatype);\n if (null == sb.parser) {\n throw new ParseException(\"Column data type not recognized (\" + sb.datatype + \")\", i);\n }\n }\n sbl.add(sb);\n }\n return sbl;\n }\n\n // Creates the DelimParser that will parse the line\n private void createDelimParser(String delimiter, int charsPerColumn,\n String nullString, String commentString,\n String skipList) throws NumberFormatException {\n delimParser = new DelimParser(delimiter, charsPerColumn, nullString, commentString);\n for (int i = 0; i < sbl.size(); i++)\n delimParser.add(sbl.get(i).parser);\n if (null != skipList) {\n for (String s : skipList.split(\",\")) {\n delimParser.addSkip(Integer.parseInt(s.trim()));\n }\n }\n jsonParser = new JSONParser();\n }\n\n // Convenience method to return the INSERT statement for a PreparedStatement.\n public String generateInsert() {\n String insert = \"INSERT INTO \" + keyspace + \".\" + tablename + \"(\" + sbl.get(0).name;\n String qmarks = \"?\";\n for (int i = 1; i < sbl.size(); i++) {\n insert = insert + \", \" + sbl.get(i).name;\n qmarks = qmarks + \", ?\";\n }\n insert = insert + \") VALUES (\" + qmarks + \")\";\n\tif (0 < ttl)\n\t insert = insert + \" USING TTL \" + ttl;\n return insert;\n }\n\n public String generateSelect() {\n String select = \"SELECT \" + sbl.get(0).name;\n for (int i = 1; i < sbl.size(); i++) {\n select = select + \", \" + sbl.get(i).name;\n }\n select += \" FROM \" + keyspace + \".\" + tablename;\n return select;\n }\n\n public String getKeyspace() {\n return keyspace;\n }\n\n public String getTable() {\n return tablename;\n }\n\n // Pass through to parse the line - the DelimParser we created will be used.\n public List parse(String line) {\n return delimParser.parse(line);\n }\n\n public List parse(String[] row) {\n return delimParser.parse(row);\n }\n\n @SuppressWarnings(\"unchecked\")\n public List parseJson(String line) {\n JSONObject jsonObject = null;\n try {\n jsonObject = (JSONObject)jsonParser.parse(line);\n } catch (org.json.simple.parser.ParseException e) {\n System.err.println(String.format(\"Invalid format in input %d: %s\",line, e.getMessage()));\n return null;\n }\n String[] row = new String[columnNames.size()];\n Set fields = (Set)jsonObject.keySet();\n for (int i = 0; i < columnNames.size(); i++) {\n String s = columnNames.get(i);\n Object o = jsonObject.get(s);\n if (null != o)\n row[i] = o.toString();\n else row[i] = null;\n fields.remove(s);\n }\n if (0 != fields.size()) {\n for (String f : fields) {\n System.err.println(\"Unknown JSON field \" + f);\n }\n return null;\n }\n return parse(row);\n }\n\n public String format(Row row) throws IndexOutOfBoundsException, InvalidTypeException {\n return delimParser.format(row);\n }\n\n public String formatJson(Row row) throws IndexOutOfBoundsException, InvalidTypeException {\n String[] stringVals = delimParser.stringVals(row);\n Map pairs = new HashMap();\n for (int i = 0; i < sbl.size(); i++)\n pairs.put(sbl.get(i).name, stringVals[i]);\n return JSONObject.toJSONString(pairs);\n }\n\n}\n\n" }, { "path": "src/main/java/com/datastax/loader/CqlDelimUnload.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport com.datastax.loader.parser.BooleanParser;\n\nimport java.util.List;\nimport java.util.ArrayList;\nimport java.util.Map;\nimport java.util.HashMap;\nimport java.util.Set;\nimport java.util.HashSet;\nimport java.util.Deque;\nimport java.util.ArrayDeque;\nimport java.util.Locale;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeUnit;\nimport java.math.BigInteger;\nimport java.io.FileOutputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.PrintStream;\nimport java.io.File;\nimport java.io.BufferedReader;\nimport java.io.FileReader;\nimport java.io.InputStreamReader;\nimport java.io.InputStream;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.io.FileNotFoundException;\nimport java.text.ParseException;\nimport java.security.KeyStore;\nimport java.security.SecureRandom;\nimport java.security.KeyStoreException;\nimport java.security.KeyManagementException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.UnrecoverableKeyException;\nimport java.security.cert.CertificateException;\nimport javax.net.ssl.KeyManagerFactory;\nimport javax.net.ssl.SSLContext;\nimport javax.net.ssl.TrustManagerFactory;\n\nimport com.datastax.driver.core.Cluster;\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.ColumnMetadata;\nimport com.datastax.driver.core.PreparedStatement;\nimport com.datastax.driver.core.PoolingOptions;\nimport com.datastax.driver.core.HostDistance;\nimport com.datastax.driver.core.BoundStatement;\nimport com.datastax.driver.core.ResultSet;\nimport com.datastax.driver.core.ConsistencyLevel;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.SSLOptions;\nimport com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions;\nimport com.datastax.driver.core.policies.TokenAwarePolicy;\nimport com.datastax.driver.core.policies.DCAwareRoundRobinPolicy;\nimport com.datastax.driver.core.exceptions.QueryValidationException;\n\n\npublic class CqlDelimUnload {\n private String version = \"0.0.27\";\n private String host = null;\n private int port = 9042;\n private String username = null;\n private String password = null;\n private String truststorePath = null;\n private String truststorePwd = null;\n private String keystorePath = null;\n private String keystorePwd = null;\n private ConsistencyLevel consistencyLevel = ConsistencyLevel.LOCAL_ONE;\n private Cluster cluster = null;\n private Session session = null;\n private String beginToken = \"-9223372036854775808\";\n private String endToken = \"9223372036854775807\";\n private String where = null;\n\n private String cqlSchema = null;\n private String filename = null;\n private String format = \"delim\";\n\n private Locale locale = null;\n private BooleanParser.BoolStyle boolStyle = null;\n private String dateFormatString = null;\n private String localDateFormatString = \"yyyy-MM-dd\";\n private String nullString = null;\n private String delimiter = null;\n\n private int numThreads = 5;\n private int fetchSize = 0;\n\n private String usage() {\n StringBuilder usage = new StringBuilder(\"version: \").append(version).append(\"\\n\");\n usage.append(\"Usage: -f -host -schema [OPTIONS]\\n\");\n usage.append(\"OPTIONS:\\n\");\n usage.append(\" -configFile File with configuration options\\n\");\n usage.append(\" -format [delim|jsonline|jsonarray] Format of data: delimited or JSON [delim]\\n\");\n usage.append(\" -delim Delimiter to use [,]\\n\");\n usage.append(\" -dateFormat Date format for TIMESTAMP [default for Locale.ENGLISH]\\n\");\n usage.append(\" -localDateFormat Date format for DATE [yyyy-MM-dd]\\n\");\n usage.append(\" -nullString String that signifies NULL [none]\\n\");\n usage.append(\" -port CQL Port Number [9042]\\n\");\n usage.append(\" -user Cassandra username [none]\\n\");\n usage.append(\" -pw Password for user [none]\\n\");\n usage.append(\" -ssl-truststore-path Path to SSL truststore [none]\\n\");\n usage.append(\" -ssl-truststore-pw Password for SSL truststore [none]\\n\");\n usage.append(\" -ssl-keystore-path Path to SSL keystore [none]\\n\");\n usage.append(\" -ssl-keystore-pw Password for SSL keystore [none]\\n\");\n usage.append(\" -consistencyLevel Consistency level [LOCAL_ONE]\\n\");\n usage.append(\" -decimalDelim Decimal delimiter [.] Other option is ','\\n\");\n usage.append(\" -boolStyle Style for booleans [TRUE_FALSE]\\n\");\n usage.append(\" -numThreads Number of concurrent threads to unload [5]\\n\");\n usage.append(\" -beginToken Begin token [none]\\n\");\n usage.append(\" -endToken End token [none]\\n\");\n usage.append(\" -where WHERE clause [none]\\n\");\n usage.append(\" -fetchSize Fetch size to use [0]\\n\");\n return usage.toString();\n }\n \n private boolean validateArgs() {\n if (!format.equalsIgnoreCase(\"delim\")\n && !format.equalsIgnoreCase(\"jsonline\")\n && !format.equalsIgnoreCase(\"jsonarray\")) {\n System.err.println(\"Invalid format (\" + format + \")\");\n return false;\n }\n if (numThreads < 1) {\n System.err.println(\"Number of threads must be positive\");\n return false;\n }\n if (fetchSize < 0) {\n System.err.println(\"Fetch size must be non-negative\");\n return false;\n }\n if ((null == username) && (null != password)) {\n System.err.println(\"If you supply the password, you must supply the username\");\n return false;\n }\n if ((null != username) && (null == password)) {\n System.err.println(\"If you supply the username, you must supply the password\");\n return false;\n }\n if (filename.equalsIgnoreCase(\"stdout\")) {\n numThreads = 1;\n }\n if ((null == truststorePath) && (null != truststorePwd)) {\n System.err.println(\"If you supply the ssl-truststore-pw, you must supply the ssl-truststore-path\");\n return false;\n }\n if ((null != truststorePath) && (null == truststorePwd)) {\n System.err.println(\"If you supply the ssl-truststore-path, you must supply the ssl-truststore-pw\");\n return false;\n }\n if ((null == keystorePath) && (null != keystorePwd)) {\n System.err.println(\"If you supply the ssl-keystore-pw, you must supply the ssl-keystore-path\");\n return false;\n }\n if ((null != keystorePath) && (null == keystorePwd)) {\n System.err.println(\"If you supply the ssl-keystore-path, you must supply the ssl-keystore-pw\");\n return false;\n }\n File tfile = null;\n if (null != truststorePath) {\n tfile = new File(truststorePath);\n if (!tfile.isFile()) {\n System.err.println(\"truststore file must be a file\");\n return false;\n }\n }\n if (null != keystorePath) {\n tfile = new File(keystorePath);\n if (!tfile.isFile()) {\n System.err.println(\"keystore file must be a file\");\n return false;\n }\n }\n if ((null != beginToken) && (null == endToken)) {\n System.err.println(\"If you supply the beginToken then you need to specify the endToken\");\n return false;\n }\n if ((null == beginToken) && (null != endToken)) {\n System.err.println(\"If you supply the endToken then you need to specify the beginToken\");\n return false;\n }\n\n return true;\n }\n \n private boolean processConfigFile(String fname, Map amap)\n throws IOException, FileNotFoundException {\n File cFile = new File(fname);\n if (!cFile.isFile()) {\n System.err.println(\"Configuration File must be a file\");\n return false;\n }\n\n BufferedReader cReader = new BufferedReader(new FileReader(cFile));\n String line;\n while ((line = cReader.readLine()) != null) {\n String[] fields = line.trim().split(\"\\\\s+\");\n if (2 != fields.length) {\n System.err.println(\"Bad line in config file: \" + line);\n return false;\n }\n if (null == amap.get(fields[0])) {\n amap.put(fields[0], fields[1]);\n }\n }\n return true;\n }\n\n private boolean parseArgs(String[] args)\n throws IOException, FileNotFoundException {\n String tkey;\n if (args.length == 0) {\n System.err.println(\"No arguments specified\");\n return false;\n }\n if (0 != args.length % 2)\n return false;\n\n Map amap = new HashMap();\n for (int i = 0; i < args.length; i+=2) {\n amap.put(args[i], args[i+1]);\n }\n\n if (null != (tkey = amap.remove(\"-configFile\")))\n if (!processConfigFile(tkey, amap))\n return false;\n\n host = amap.remove(\"-host\");\n if (null == host) { // host is required\n System.err.println(\"Must provide a host\");\n return false;\n }\n\n filename = amap.remove(\"-f\");\n if (null == filename) { // filename is required\n System.err.println(\"Must provide an output filename stem\");\n return false;\n }\n\n cqlSchema = amap.remove(\"-schema\");\n if (null == cqlSchema) { // schema is required\n System.err.println(\"Must provide a schema\");\n return false;\n }\n\n if (null != (tkey = amap.remove(\"-port\"))) port = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-user\"))) username = tkey;\n if (null != (tkey = amap.remove(\"-pw\"))) password = tkey;\n if (null != (tkey = amap.remove(\"-ssl-truststore-path\"))) truststorePath = tkey;\n if (null != (tkey = amap.remove(\"-ssl-truststore-pw\"))) truststorePwd = tkey;\n if (null != (tkey = amap.remove(\"-ssl-keystore-path\"))) keystorePath = tkey;\n if (null != (tkey = amap.remove(\"-ssl-keystore-pw\"))) keystorePwd = tkey;\n if (null != (tkey = amap.remove(\"-consistencyLevel\"))) consistencyLevel = ConsistencyLevel.valueOf(tkey);\n if (null != (tkey = amap.remove(\"-dateFormat\"))) dateFormatString = tkey;\n if (null != (tkey = amap.remove(\"-localDateFormat\"))) localDateFormatString = tkey;\n if (null != (tkey = amap.remove(\"-nullString\"))) nullString = tkey;\n if (null != (tkey = amap.remove(\"-delim\"))) delimiter = tkey;\n if (null != (tkey = amap.remove(\"-decimalDelim\"))) {\n if (tkey.equals(\",\"))\n locale = Locale.FRANCE;\n }\n if (null != (tkey = amap.remove(\"-boolStyle\"))) {\n boolStyle = BooleanParser.getBoolStyle(tkey);\n if (null == boolStyle) {\n System.err.println(\"Bad boolean style. Options are: \" + BooleanParser.getOptions());\n return false;\n }\n }\n if (null != (tkey = amap.remove(\"-numThreads\"))) numThreads = Integer.parseInt(tkey);\n if (null != (tkey = amap.remove(\"-beginToken\"))) beginToken = tkey;\n if (null != (tkey = amap.remove(\"-endToken\"))) endToken = tkey;\n if (null != (tkey = amap.remove(\"-where\"))) where = tkey;\n if (null != (tkey = amap.remove(\"-format\"))) format = tkey;\n if (null != (tkey = amap.remove(\"-fetchSize\"))) fetchSize = Integer.parseInt(tkey);\n \n if (!amap.isEmpty()) {\n for (String k : amap.keySet())\n System.err.println(\"Unrecognized option: \" + k);\n return false;\n }\n return validateArgs();\n }\n\n private SSLOptions createSSLOptions()\n throws KeyStoreException, FileNotFoundException, IOException, NoSuchAlgorithmException,\n KeyManagementException, CertificateException, UnrecoverableKeyException {\n TrustManagerFactory tmf = null;\n KeyStore tks = KeyStore.getInstance(\"JKS\");\n tks.load((InputStream) new FileInputStream(new File(truststorePath)),\n truststorePwd.toCharArray());\n tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());\n tmf.init(tks);\n\n KeyManagerFactory kmf = null;\n if (null != keystorePath) {\n KeyStore kks = KeyStore.getInstance(\"JKS\");\n kks.load((InputStream) new FileInputStream(new File(keystorePath)),\n keystorePwd.toCharArray());\n kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());\n kmf.init(kks, keystorePwd.toCharArray());\n }\n\n SSLContext sslContext = SSLContext.getInstance(\"TLS\");\n sslContext.init(kmf != null? kmf.getKeyManagers() : null,\n tmf != null ? tmf.getTrustManagers() : null,\n new SecureRandom());\n\n return RemoteEndpointAwareJdkSSLOptions.builder().withSSLContext(sslContext).build();\n }\n\n private void setup()\n throws IOException, KeyStoreException, NoSuchAlgorithmException, KeyManagementException,\n CertificateException, UnrecoverableKeyException {\n // Connect to Cassandra\n PoolingOptions pOpts = new PoolingOptions();\n pOpts.setCoreConnectionsPerHost(HostDistance.LOCAL, 4);\n pOpts.setMaxConnectionsPerHost(HostDistance.LOCAL, 4);\n Cluster.Builder clusterBuilder = Cluster.builder()\n .addContactPoint(host)\n .withPort(port)\n .withPoolingOptions(pOpts)\n .withLoadBalancingPolicy(new TokenAwarePolicy( DCAwareRoundRobinPolicy.builder().build()));\n if (null != username)\n clusterBuilder = clusterBuilder.withCredentials(username, password);\n if (null != truststorePath)\n clusterBuilder = clusterBuilder.withSSL(createSSLOptions());\n\n cluster = clusterBuilder.build();\n if (null == cluster) {\n throw new IOException(\"Could not create cluster\");\n }\n session = cluster.connect();\n }\n\n private void cleanup() {\n if (null != session)\n session.close();\n if (null != cluster)\n cluster.close();\n }\n \n public boolean run(String[] args) \n throws IOException, ParseException, InterruptedException, ExecutionException,\n KeyStoreException, NoSuchAlgorithmException, KeyManagementException,\n CertificateException, UnrecoverableKeyException {\n if (false == parseArgs(args)) {\n System.err.println(\"Bad arguments\");\n System.err.println(usage());\n return false;\n }\n\n // Setup\n setup();\n\n PrintStream pstream = null;\n if (1 == numThreads) {\n if (filename.equalsIgnoreCase(\"stdout\")) {\n pstream = System.out;\n }\n else {\n pstream = new PrintStream(new BufferedOutputStream(new FileOutputStream(filename + \".0\")));\n }\n beginToken = null;\n endToken = null;\n }\n \n // Launch Threads\n ExecutorService executor;\n long total = 0;\n if (null != pstream) {\n // One file/stdin to process\n executor = Executors.newSingleThreadExecutor();\n Callable worker = new ThreadExecute(cqlSchema, delimiter, \n nullString,\n dateFormatString, \n localDateFormatString, \n boolStyle, locale, \n pstream, \n beginToken,\n endToken, session,\n consistencyLevel, where,\n format, fetchSize);\n Future res = executor.submit(worker);\n total = res.get();\n executor.shutdown();\n }\n else {\n BigInteger begin = null;\n BigInteger end = null;\n BigInteger delta = null;\n List beginList = new ArrayList();\n List endList = new ArrayList();\n if (null != beginToken) {\n begin = new BigInteger(beginToken);\n end = new BigInteger(endToken);\n delta = end.subtract(begin).divide(new BigInteger(String.valueOf(numThreads)));\n for (int mype = 0; mype < numThreads; mype++) {\n if (mype < numThreads - 1) {\n beginList.add(begin.add(delta.multiply(new BigInteger(String.valueOf(mype)))).toString());\n endList.add(begin.add(delta.multiply(new BigInteger(String.valueOf(mype+1)))).toString());\n }\n else {\n beginList.add(begin.add(delta.multiply(new BigInteger(String.valueOf(numThreads-1)))).toString());\n endList.add(end.toString());\n }\n }\n }\n else {\n // What's the right thing here?\n // (1) Split into canonical token ranges - numThreads=numRanges\n // (2) Split into subranges of canonical token ranges\n // - if numThreads < numRanges, then reset numThreads=numRanges\n // - let K=CEIL(numThreads/numRanges) and M=MOD(numThreads/numRanges), for the first M token ranges split into K subranges, and for the remaining ones split into K-1 subranges\n // (?) Should there be an option for numThreads-per-range?\n // (?) Should there be an option for numThreads=numRanges\n }\n\n executor = Executors.newFixedThreadPool(numThreads);\n Set> results = new HashSet>();\n for (int mype = 0; mype < numThreads; mype++) {\n String tBeginString = beginList.get(mype);\n String tEndString = endList.get(mype);\n pstream = new PrintStream(new BufferedOutputStream(new FileOutputStream(filename + \".\" + mype)));\n Callable worker = new ThreadExecute(cqlSchema, delimiter, \n nullString,\n dateFormatString, \n localDateFormatString, \n boolStyle, locale, \n pstream, \n tBeginString,\n tEndString, session,\n consistencyLevel,\n where, format, fetchSize);\n results.add(executor.submit(worker));\n }\n executor.shutdown();\n for (Future res : results)\n total += res.get();\n }\n System.err.println(\"Total rows retrieved: \" + total);\n\n // Cleanup\n cleanup();\n\n return true;\n }\n\n public static void main(String[] args) \n throws IOException, ParseException, InterruptedException, ExecutionException,\n KeyStoreException, NoSuchAlgorithmException, UnrecoverableKeyException,\n CertificateException, KeyManagementException {\n CqlDelimUnload cdu = new CqlDelimUnload();\n boolean success = cdu.run(args);\n if (success) {\n System.exit(0);\n } else {\n System.exit(-1);\n }\n }\n\n class ThreadExecute implements Callable {\n private Session session;\n private ConsistencyLevel consistencyLevel;\n private PreparedStatement statement;\n private CqlDelimParser cdp;\n\n private String cqlSchema;\n private String format = \"delim\";\n private Locale locale = null;\n private BooleanParser.BoolStyle boolStyle = null;\n private String nullString = null;\n private String delimiter = null;\n\n private PrintStream writer = null;\n private String beginToken = null;\n private String endToken = null;\n private String partitionKey = null;\n private long numRead = 0;\n private String where = null;\n private String dateFormatString = null;\n private String localDateFormatString = null;\n private int fetchSize = 0;\n\n public ThreadExecute(String inCqlSchema, String inDelimiter, \n String inNullString, \n String inDateFormatString,\n String inLocalDateFormatString,\n BooleanParser.BoolStyle inBoolStyle, \n Locale inLocale, \n PrintStream inWriter,\n String inBeginToken, String inEndToken,\n Session inSession, ConsistencyLevel inConsistencyLevel,\n String inWhere, String inFormat, int inFetchSize) {\n super();\n cqlSchema = inCqlSchema;\n delimiter = inDelimiter;\n nullString = inNullString;\n dateFormatString = inDateFormatString;\n localDateFormatString = inLocalDateFormatString;\n boolStyle = inBoolStyle;\n locale = inLocale;\n beginToken = inBeginToken;\n endToken = inEndToken;\n session = inSession;\n writer = inWriter;\n consistencyLevel = inConsistencyLevel;\n where = inWhere;\n format = inFormat;\n fetchSize = inFetchSize;\n }\n\n public Long call() throws IOException, ParseException {\n if (false == setup()) {\n return 0L;\n }\n numRead = execute();\n cleanup();\n return numRead;\n }\n\n private String getPartitionKey(CqlDelimParser cdp, Session session) {\n String keyspace = cdp.getKeyspace();\n String table = cdp.getTable();\n if (keyspace.startsWith(\"\\\"\") && keyspace.endsWith(\"\\\"\"))\n keyspace = keyspace.replaceAll(\"\\\"\", \"\");\n else\n keyspace = keyspace.toLowerCase();\n if (table.startsWith(\"\\\"\") && table.endsWith(\"\\\"\"))\n table = table.replaceAll(\"\\\"\", \"\");\n else\n table = table.toLowerCase();\n\n List lcm = session.getCluster().getMetadata()\n .getKeyspace(keyspace).getTable(table).getPartitionKey();\n String partitionKey = lcm.get(0).getName();\n for (int i = 1; i < lcm.size(); i++) {\n partitionKey = partitionKey + \",\" + lcm.get(i).getName();\n }\n return partitionKey;\n }\n\n private boolean setup() throws IOException, ParseException {\n cdp = new CqlDelimParser(cqlSchema, delimiter, 4096, nullString, \n null, dateFormatString, localDateFormatString,\n boolStyle, locale, null, session, false, -1);\n String select = cdp.generateSelect();\n String partitionKey = getPartitionKey(cdp, session);\n if (null != beginToken) {\n select = select + \" WHERE Token(\" + partitionKey + \") > \" \n + beginToken + \" AND Token(\" + partitionKey + \") <= \" \n + endToken;\n if (null != where)\n select = select + \" AND \" + where;\n }\n else {\n if (null != where)\n select = select + \" WHERE \" + where;\n }\n try {\n statement = session.prepare(select);\n }\n catch (QueryValidationException iqe) {\n System.err.println(\"Error creating statement: \" + iqe.getMessage());\n System.err.println(\"CQL Query: \" + select);\n if (null != where)\n System.err.println(\"Check your syntax for -where: \" + where);\n return false;\n }\n statement.setConsistencyLevel(consistencyLevel);\n return true;\n }\n \n private void cleanup() throws IOException {\n writer.flush();\n writer.close();\n }\n\n private long execute() throws IOException {\n BoundStatement bound = statement.bind();\n bound.setFetchSize(fetchSize); \n ResultSet rs = session.execute(bound);\n numRead = 0;\n String s = null;\n String jsonArrayChar = \"[\\n\";\n for (Row row : rs) {\n if (format.equalsIgnoreCase(\"jsonarray\")) {\n writer.print(jsonArrayChar);\n jsonArrayChar = \",\";\n }\n if (format.equalsIgnoreCase(\"delim\")) {\n s = cdp.format(row);\n }\n else if (format.equalsIgnoreCase(\"jsonline\")\n || format.equalsIgnoreCase(\"jsonarray\")) {\n s = cdp.formatJson(row);\n }\n writer.println(s);\n numRead++;\n }\n if (format.equalsIgnoreCase(\"jsonarray\"))\n writer.println(\"]\");\n return numRead;\n }\n }\n}\n\n" }, { "path": "src/main/java/com/datastax/loader/EnhancedSession.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport java.util.Map;\n\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.Cluster;\nimport com.datastax.driver.core.CloseFuture;\nimport com.datastax.driver.core.ResultSet;\nimport com.datastax.driver.core.ResultSetFuture;\nimport com.datastax.driver.core.Statement;\nimport com.datastax.driver.core.RegularStatement;\nimport com.datastax.driver.core.PreparedStatement;\n\nimport com.google.common.util.concurrent.ListenableFuture;\n\npublic class EnhancedSession implements Session {\n private Session session;\n public EnhancedSession(Session inSession) {\n session = inSession;\n }\n\n public void close() {\n session.close();\n }\n\n public CloseFuture closeAsync() {\n return session.closeAsync();\n }\n\n public ResultSet execute(Statement statement) {\n return session.execute(statement);\n }\n\n public ResultSet execute(String query) {\n return session.execute(query);\n }\n\n public ResultSet execute(String query, Object... values) {\n return session.execute(query, values);\n }\n\n public ResultSet execute(String query, Map values) {\n return session.execute(query, values);\n }\n\n public ResultSetFuture executeAsync(Statement statement) {\n return session.executeAsync(statement);\n }\n\n public ResultSetFuture executeAsync(String query) {\n return session.executeAsync(query);\n }\n\n public ResultSetFuture executeAsync(String query, Object... values) {\n return session.executeAsync(query, values);\n }\n\n public ResultSetFuture executeAsync(String query, Map values) {\n return session.executeAsync(query, values);\n }\n\n public Cluster getCluster() {\n return session.getCluster();\n }\n\n public String getLoggedKeyspace() {\n return session.getLoggedKeyspace();\n }\n\n public Session.State getState() {\n return session.getState();\n }\n\n public EnhancedSession init() {\n session.init();\n return this;\n }\n\n public com.google.common.util.concurrent.ListenableFuture initAsync() {\n return session.initAsync();\n }\n\n public boolean isClosed() {\n return session.isClosed();\n }\n\n public PreparedStatement prepare(RegularStatement statement) {\n return session.prepare(statement);\n }\n\n public PreparedStatement prepare(String query) {\n return session.prepare(query);\n }\n\n public com.google.common.util.concurrent.ListenableFuture\n prepareAsync(RegularStatement statement) {\n return session.prepareAsync(statement);\n }\n\n public com.google.common.util.concurrent.ListenableFuture\n prepareAsync(String query) {\n return session.prepareAsync(query);\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/Latency999RateLimiter.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport com.datastax.driver.core.Cluster;\n\npublic class Latency999RateLimiter extends AbstractDynamicRateLimiter {\n private Cluster cluster;\n\n public Latency999RateLimiter(double inRate, long inHowOften,\n double inMaxStat, double inMinStat,\n double inDownFraction, \n double inUpFraction, Cluster inCluster,\n boolean inInvertLogic) {\n this(inRate, Long.MAX_VALUE, inHowOften, inMaxStat, inMinStat,\n inDownFraction, inUpFraction, inCluster, inInvertLogic);\n }\n\n public Latency999RateLimiter(double inRate, long inUpdateRate,\n long inHowOften, double inMaxStat, \n double inMinStat, double inDownFraction, \n double inUpFraction, Cluster inCluster,\n boolean inInvertLogic) {\n super(inRate, inUpdateRate, inHowOften, inMaxStat, inMinStat, \n inDownFraction, inUpFraction, inInvertLogic);\n cluster = inCluster;\n }\n\n protected synchronized double getCurrStat() {\n return cluster.getMetrics().getRequestsTimer().getSnapshot().get999thPercentile();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/LoaderRetryPolicy.java", "content": "package com.datastax.loader;\n\nimport java.io.File;\nimport java.io.BufferedWriter;\nimport java.io.FileWriter;\n\nimport com.datastax.driver.core.policies.RetryPolicy;\nimport com.datastax.driver.core.ConsistencyLevel;\nimport com.datastax.driver.core.Statement;\nimport com.datastax.driver.core.Cluster;\nimport com.datastax.driver.core.WriteType;\nimport com.datastax.driver.core.exceptions.DriverException;\n\nclass LoaderRetryPolicy implements RetryPolicy {\n private int numRetries;\n\n public LoaderRetryPolicy(int inNumRetries) {\n numRetries = inNumRetries;\n }\n\n // Taken from DefaultRetryPolicy\n public RetryDecision onReadTimeout(Statement statement, ConsistencyLevel cl,\n int requiredResponses, \n int receivedResponses, \n boolean dataRetrieved, int nbRetry) {\n if (nbRetry != 0)\n return RetryDecision.rethrow();\n\n return receivedResponses >= requiredResponses && !dataRetrieved \n ? RetryDecision.retry(cl) \n : RetryDecision.rethrow();\n }\n\n // Taken from DefaultRetryPolicy\n public RetryDecision onUnavailable(Statement statement, ConsistencyLevel cl,\n int requiredReplica, int aliveReplica, \n int nbRetry) {\n return RetryDecision.rethrow();\n }\n\n public RetryDecision onWriteTimeout(Statement statement, \n ConsistencyLevel cl, \n WriteType writeType, int requiredAcks, \n int receivedAcks, int nbRetry) {\n if (nbRetry >= numRetries)\n return RetryDecision.rethrow();\n\n return RetryDecision.retry(cl);\n }\n\n public RetryPolicy.RetryDecision onRequestError(Statement statement,\n ConsistencyLevel cl,\n DriverException e,\n int nbRetry) {\n return RetryDecision.tryNextHost(cl);\n }\n\n public void close() {\n }\n\n public void init(Cluster cluster) {\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/RateLimitedSession.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport com.datastax.driver.core.Session;\nimport com.datastax.driver.core.ResultSet;\nimport com.datastax.driver.core.ResultSetFuture;\nimport com.datastax.driver.core.Statement;\nimport com.datastax.driver.core.BatchStatement;\n\npublic class RateLimitedSession extends EnhancedSession {\n RateLimiter rateLimiter;\n private Class batchClass;\n public RateLimitedSession(Session inSession, RateLimiter inRateLimiter) {\n super(inSession);\n rateLimiter = inRateLimiter;\n BatchStatement batch = new BatchStatement();\n batchClass = batch.getClass();\n }\n\n public long numAcquires() {\n return rateLimiter.numAcquires();\n }\n\n public double getRate() {\n return rateLimiter.getRate();\n }\n\n public ResultSet execute(Statement statement) {\n rateLimiter.acquire();\n return super.execute(statement);\n }\n\n public ResultSet execute(String query) {\n rateLimiter.acquire();\n return super.execute(query);\n }\n\n public ResultSet execute(String query, Object... values) {\n rateLimiter.acquire();\n return super.execute(query, values);\n }\n\n public ResultSetFuture executeAsync(Statement statement) {\n if (statement.getClass() == batchClass)\n rateLimiter.acquire(((BatchStatement)statement).size());\n else\n rateLimiter.acquire();\n return super.executeAsync(statement);\n }\n\n public ResultSetFuture executeAsync(String query) {\n rateLimiter.acquire();\n return super.executeAsync(query);\n }\n\n public ResultSetFuture executeAsync(String query, Object... values) {\n rateLimiter.acquire();\n return super.executeAsync(query, values);\n }\n\n}\n" }, { "path": "src/main/java/com/datastax/loader/RateLimiter.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader;\n\nimport java.io.PrintStream;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport com.codahale.metrics.Timer;\nimport com.codahale.metrics.Snapshot;\n\npublic class RateLimiter {\n private com.google.common.util.concurrent.RateLimiter rateLimiter;\n private AtomicLong numAcquires;\n private static long updateRate = 100000;\n private Timer timer;\n private PrintStream stream;\n private long lastVal;\n private long firstTime;\n private long lastTime;\n\n public RateLimiter(double inRate) {\n this(inRate, Long.MAX_VALUE);\n }\n\n public RateLimiter(double inRate, long inUpdateRate) {\n this(inRate, inUpdateRate, null, null);\n }\n\n public RateLimiter(double inRate, long inUpdateRate,\n Timer inTimer, PrintStream inStream) {\n rateLimiter = com.google.common.util.concurrent.RateLimiter.create(inRate);\n updateRate = inUpdateRate;\n timer = inTimer;\n stream = inStream;\n if ((null != stream) && (null != timer)) {\n printHeader();\n }\n numAcquires = new AtomicLong(0);\n lastTime = System.currentTimeMillis();\n firstTime = lastTime;\n lastVal = 0;\n }\n\n protected void printHeader() {\n stream.println(\"Count,Min,Max,Mean,StdDev,50th,75th,95th,98th,99th,999th,MeanRate,1MinuteRate,5MinuteRate,15MinuteRate\");\n }\n\n protected void printStats() {\n Snapshot snapshot = timer.getSnapshot();\n stream.println(String.format(\"%d,%d,%d,%.4f,%.4f,%.4f,%.4f,%.4f,%.4f,%.4f,%.4f,%.4f,%.4f,%.4f,%.4f\",\n timer.getCount(),\n snapshot.getMin(),\n snapshot.getMax(),\n snapshot.getMean(),\n snapshot.getStdDev(),\n snapshot.getMedian(),\n snapshot.get75thPercentile(),\n snapshot.get95thPercentile(),\n snapshot.get98thPercentile(),\n snapshot.get99thPercentile(),\n snapshot.get999thPercentile(),\n timer.getMeanRate(),\n timer.getOneMinuteRate(),\n timer.getFiveMinuteRate(),\n timer.getFifteenMinuteRate())\n );\n }\n\n public void report(Long currentVal, Long currentTime) {\n if ((null != stream) && (null != timer)) {\n printStats();\n return;\n }\n if (null == currentTime)\n currentTime = System.currentTimeMillis();\n long etime = (currentTime - firstTime)/1000;\n double rateFromBeginning;\n if (null == currentVal) {\n currentVal = numAcquires.get() - 1;\n rateFromBeginning = (etime > 0) ? (currentVal + 0.0) / etime : 0;\n System.err.println(\"Lines Processed: \\t\" + currentVal \n + \" Rate: \\t\" + rateFromBeginning);\n }\n else {\n long ltime = (currentTime - lastTime)/1000;\n double rateFromLast = (ltime > 0) ? (currentVal - lastVal + 0.0) / ltime : 0;\n rateFromBeginning = (etime > 0) ? (currentVal + 0.0) / etime : 0;\n System.err.println(\"Lines Processed: \\t\" + currentVal \n + \" Rate: \\t\" + rateFromBeginning\n + \" (\" + rateFromLast\n + \")\"\n );\n }\n }\n\n protected synchronized void incrementAndReport(int permits) {\n long currentVal = numAcquires.addAndGet(permits);\n long currentTime = System.currentTimeMillis();\n if (permits > currentVal % updateRate) {\n report(currentVal, currentTime);\n lastTime = currentTime;\n lastVal = currentVal;\n }\n }\n\n public void acquire() {\n rateLimiter.acquire();\n incrementAndReport(1);\n }\n\n public void acquire(int permits) {\n rateLimiter.acquire(permits);\n incrementAndReport(permits);\n }\n\n public double getRate() {\n return rateLimiter.getRate();\n }\n\n public synchronized void setRate(double permitsPerSecond) {\n rateLimiter.setRate(permitsPerSecond);\n }\n\n public String toString() {\n return rateLimiter.toString();\n }\n\n public boolean tryAcquire() {\n if (rateLimiter.tryAcquire()) {\n incrementAndReport(1);\n return true;\n }\n return false;\n }\n\n public boolean tryAcquire(int permits) {\n if (rateLimiter.tryAcquire(permits)) {\n incrementAndReport(permits);\n return true;\n }\n return false;\n }\n\n public boolean tryAcquire(int permits, long timeout, TimeUnit unit) {\n if (rateLimiter.tryAcquire(permits, timeout, unit)) {\n incrementAndReport(permits);\n return true;\n }\n return false;\n }\n\n public boolean tryAcquire(long timeout, TimeUnit unit) {\n if (rateLimiter.tryAcquire(timeout, unit)) {\n incrementAndReport(1);\n return true;\n }\n return false;\n }\n\n public long numAcquires() {\n return numAcquires.get();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/AbstractFutureManager.java", "content": "package com.datastax.loader.futures;\n\nimport java.util.concurrent.TimeUnit;\nimport com.datastax.driver.core.ResultSetFuture;\n\npublic abstract class AbstractFutureManager implements FutureManager {\n protected int size;\n protected long queryTimeout;\n protected long maxInsertErrors;\n protected TimeUnit unit = TimeUnit.SECONDS;\n\n public AbstractFutureManager(int inSize, long inQueryTimeout, long inMaxInsertErrors) {\n size = inSize;\n queryTimeout = inQueryTimeout;\n maxInsertErrors = inMaxInsertErrors;\n }\n\n public abstract boolean add(ResultSetFuture future, String line);\n\n public abstract boolean cleanup();\n\n public abstract long getNumInserted();\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/ActionFutureList.java", "content": "package com.datastax.loader.futures;\n\nimport java.util.List;\nimport java.util.ArrayList;\n\nimport com.datastax.driver.core.ResultSetFuture;\nimport com.datastax.driver.core.ResultSet;\n\npublic class ActionFutureList extends AbstractFutureManager {\n protected List futures;\n protected List strings;\n protected long insertErrors;\n protected long numInserted;\n protected FutureAction futureAction = null;\n\n public ActionFutureList(int inSize, long inQueryTimeout, long inMaxInsertErrors, FutureAction inFutureAction) {\n super(inSize, inQueryTimeout, inMaxInsertErrors);\n futureAction = inFutureAction;\n futures = new ArrayList(size);\n strings = new ArrayList(size);\n insertErrors = 0;\n numInserted = 0;\n }\n \n public boolean add(ResultSetFuture future, String line) {\n if (futures.size() >= size) {\n if (!purgeFutures())\n return false;\n }\n futures.add(future);\n strings.add(line);\n numInserted++;\n return true;\n }\n\n protected boolean purgeFutures() {\n if (0 == futures.size())\n return true;\n for (int i = 0; i < futures.size(); i++) {\n ResultSetFuture future = futures.get(i);\n String line = strings.get(i);\n try {\n //long beginTime = System.currentTimeMillis();\n ResultSet rs = future.getUninterruptibly(queryTimeout, unit);\n //long duration = System.currentTimeMillis() - beginTime;\n //if (2000 < duration) {\n //System.err.println(\"Query took \" + duration + \" ms\");\n //}\n futureAction.onSuccess(rs, line);\n }\n catch (Exception e) {\n insertErrors++;\n futureAction.onFailure(e, line);\n if (maxInsertErrors <= insertErrors) {\n futureAction.onTooManyFailures();\n return false;\n }\n }\n }\n futures.clear();\n return true;\n }\n\n public boolean cleanup() {\n return purgeFutures();\n }\n\n public long getNumInserted() {\n return numInserted;\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/ActionFutureSet.java", "content": "package com.datastax.loader.futures;\n\nimport java.util.List;\nimport java.util.ArrayList;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport com.datastax.driver.core.ResultSetFuture;\nimport com.datastax.driver.core.ResultSet;\n\nimport com.google.common.util.concurrent.Futures;\nimport com.google.common.util.concurrent.FutureCallback;\n\npublic class ActionFutureSet extends AbstractFutureManager {\n protected FutureAction futureAction = null;\n protected Semaphore available;\n protected AtomicLong insertErrors;\n protected AtomicLong numInserted;\n\n public ActionFutureSet(int inSize, long inQueryTimeout, \n long inMaxInsertErrors, \n FutureAction inFutureAction) {\n super(inSize, inQueryTimeout, inMaxInsertErrors);\n futureAction = inFutureAction;\n available = new Semaphore(size, true);\n insertErrors = new AtomicLong(0);\n numInserted = new AtomicLong(0);\n }\n\n public boolean add(ResultSetFuture future, final String line) {\n if (maxInsertErrors <= insertErrors.get())\n return false;\n try {\n available.acquire();\n }\n catch (InterruptedException e) {\n return false;\n }\n Futures.addCallback(future, new FutureCallback() {\n @Override\n public void onSuccess(ResultSet rs) {\n available.release();\n numInserted.incrementAndGet();\n futureAction.onSuccess(rs, line);\n }\n @Override\n public void onFailure(Throwable t) {\n available.release();\n long numErrors = insertErrors.incrementAndGet();\n futureAction.onFailure(t, line);\n if (maxInsertErrors <= numErrors) {\n futureAction.onTooManyFailures();\n }\n }\n });\n return true;\n }\n\n public boolean cleanup() {\n try {\n available.acquire(this.size);\n } catch (InterruptedException e) {\n return false;\n }\n return true;\n }\n\n public long getNumInserted() {\n return numInserted.get();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/FutureAction.java", "content": "package com.datastax.loader.futures;\n\nimport com.datastax.driver.core.ResultSet;\n\npublic interface FutureAction {\n public void onSuccess(ResultSet rs, String line);\n public void onFailure(Throwable t, String line);\n public void onTooManyFailures();\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/FutureManager.java", "content": "package com.datastax.loader.futures;\n\nimport com.datastax.driver.core.ResultSetFuture;\n\npublic interface FutureManager {\n public boolean add(ResultSetFuture future, String line);\n\n public boolean cleanup();\n\n public long getNumInserted();\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/JsonPrintingFutureAction.java", "content": "package com.datastax.loader.futures;\n\nimport java.io.PrintStream;\nimport java.util.concurrent.atomic.AtomicLong;\nimport com.datastax.driver.core.ResultSet;\n\npublic class JsonPrintingFutureAction extends PrintingFutureAction {\n private boolean firstBad = true;\n private String badDelim = \"[\\n\";\n public JsonPrintingFutureAction(PrintStream inLogPrinter, \n PrintStream inBadInsertPrinter) {\n super(inLogPrinter, inBadInsertPrinter);\n }\n \n public void onFailure(Throwable t, String line) {\n if (logPrinter != null) {\n logPrinter.println(\"Error inserting: \" + t.getMessage());\n t.printStackTrace(logPrinter);\n }\n if (badInsertPrinter != null) {\n badInsertPrinter.println(badDelim + line);\n if (firstBad) {\n firstBad = false;\n badDelim = \",\\n\";\n }\n }\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/JsonPrintingFutureList.java", "content": "package com.datastax.loader.futures;\n\nimport java.io.PrintStream;\n\nimport com.datastax.driver.core.ResultSetFuture;\n\npublic class JsonPrintingFutureList extends ActionFutureList {\n public JsonPrintingFutureList() {\n this(500, 2, 10);\n }\n\n public JsonPrintingFutureList(int inSize, long inQueryTimeout, \n long inMaxInsertErrors) {\n this(inSize, inQueryTimeout, inMaxInsertErrors, System.err, System.err);\n }\n \n public JsonPrintingFutureList(int inSize, long inQueryTimeout, \n long inMaxInsertErrors, \n PrintStream inLogPrinter, \n PrintStream inBadInsertPrinter) {\n super(inSize, inQueryTimeout, inMaxInsertErrors, \n new JsonPrintingFutureAction(inLogPrinter, inBadInsertPrinter));\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/JsonPrintingFutureSet.java", "content": "package com.datastax.loader.futures;\n\nimport java.io.PrintStream;\n\npublic class JsonPrintingFutureSet extends ActionFutureSet {\n public JsonPrintingFutureSet() {\n this(500, 2, 10);\n }\n\n public JsonPrintingFutureSet(int inSize, long inQueryTimeout, \n long inMaxInsertErrors) {\n this(inSize, inQueryTimeout, inMaxInsertErrors, System.err, System.err);\n }\n \n public JsonPrintingFutureSet(int inSize, long inQueryTimeout, \n long inMaxInsertErrors, \n PrintStream inLogPrinter, \n PrintStream inBadInsertPrinter) {\n super(inSize, inQueryTimeout, inMaxInsertErrors, \n new JsonPrintingFutureAction(inLogPrinter, inBadInsertPrinter));\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/NullFutureAction.java", "content": "package com.datastax.loader.futures;\n\nimport com.datastax.driver.core.ResultSet;\n\npublic class NullFutureAction implements FutureAction {\n public void onSuccess(ResultSet rs, String line) { }\n public void onFailure(Throwable t, String line) { }\n public void onTooManyFailures() { }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/PrintingFutureAction.java", "content": "package com.datastax.loader.futures;\n\nimport java.io.PrintStream;\nimport java.util.concurrent.atomic.AtomicLong;\nimport com.datastax.driver.core.ResultSet;\n\npublic class PrintingFutureAction implements FutureAction {\n protected PrintStream logPrinter = null;\n protected PrintStream badInsertPrinter = null;\n protected AtomicLong numInserted;\n protected final long period = 100000;\n\n public PrintingFutureAction(PrintStream inLogPrinter, \n PrintStream inBadInsertPrinter) {\n logPrinter = inLogPrinter;\n badInsertPrinter = inBadInsertPrinter;\n numInserted = new AtomicLong(0);\n }\n \n public void onSuccess(ResultSet rs, String line) {\n if (logPrinter != null) {\n long cur = numInserted.incrementAndGet();\n if (0 == (cur % period)) {\n logPrinter.println(\"Progress: \" + cur);\n }\n } \n }\n\n public void onFailure(Throwable t, String line) {\n if (logPrinter != null) {\n logPrinter.println(\"Error inserting: \" + t.getMessage());\n t.printStackTrace(logPrinter);\n }\n if (badInsertPrinter != null) {\n badInsertPrinter.println(line);\n }\n }\n\n public void onTooManyFailures() {\n if (logPrinter != null) {\n logPrinter.println(\"Too many INSERT errors ... Stopping\");\n }\n }\n\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/PrintingFutureList.java", "content": "package com.datastax.loader.futures;\n\nimport java.io.PrintStream;\n\nimport com.datastax.driver.core.ResultSetFuture;\n\npublic class PrintingFutureList extends ActionFutureList {\n public PrintingFutureList() {\n this(500, 2, 10);\n }\n\n public PrintingFutureList(int inSize, long inQueryTimeout, long inMaxInsertErrors) {\n this(inSize, inQueryTimeout, inMaxInsertErrors, System.err, System.err);\n }\n \n public PrintingFutureList(int inSize, long inQueryTimeout, long inMaxInsertErrors, \n PrintStream inLogPrinter, PrintStream inBadInsertPrinter) {\n super(inSize, inQueryTimeout, inMaxInsertErrors, new PrintingFutureAction(inLogPrinter, inBadInsertPrinter));\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/futures/PrintingFutureSet.java", "content": "package com.datastax.loader.futures;\n\nimport java.io.PrintStream;\n\npublic class PrintingFutureSet extends ActionFutureSet {\n public PrintingFutureSet() {\n this(500, 2, 10);\n }\n\n public PrintingFutureSet(int inSize, long inQueryTimeout, \n long inMaxInsertErrors) {\n this(inSize, inQueryTimeout, inMaxInsertErrors, System.err, System.err);\n }\n \n public PrintingFutureSet(int inSize, long inQueryTimeout, \n long inMaxInsertErrors, \n PrintStream inLogPrinter, \n PrintStream inBadInsertPrinter) {\n super(inSize, inQueryTimeout, inMaxInsertErrors, \n new PrintingFutureAction(inLogPrinter, inBadInsertPrinter));\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/AbstractParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.io.StringReader;\nimport java.io.IOException;\nimport java.text.ParseException;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\nimport org.apache.commons.lang3.StringEscapeUtils;\n\npublic abstract class AbstractParser implements Parser {\n public abstract Object parseIt(String toparse) throws ParseException;\n public String format(Row row, int index) throws IndexOutOfBoundsException, InvalidTypeException {\n if (row.isNull(index))\n return null;\n return format(row.getObject(index));\n }\n public abstract String format(Object o);\n\n public Object parse(String toparse) throws ParseException {\n String toparseit = unquote(toparse);\n return parseIt(toparseit);\n }\n\n public Object parse(IndexedLine il, String nullString, Character delim, \n Character escape, Character quote, boolean last)\n throws IOException, ParseException {\n //if (last)\n // return parse(prepareToParse(il.remaining(), nullString, quote));\n //return parse(getQuotedOrUnquoted(il, nullString, delim, escape, quote));\n return parse(getQuotedOrUnquoted(il, nullString, delim, escape, quote));\n }\n\n public String prepareToParse(String retstring, String nullString, Character quote) {\n if (retstring.startsWith(quote.toString()) \n && retstring.endsWith(quote.toString()))\n //if ((quote == retstring.charAt(0)) \n //&& (quote == retstring.charAt(retstring.length() - 1)))\n retstring = retstring.substring(1, retstring.length() - 1);\n else \n retstring = retstring.trim();\n if (null != nullString)\n if (nullString.equalsIgnoreCase(retstring))\n return null; \n return retstring;\n }\n\n public String getQuotedOrUnquoted(IndexedLine il, String nullString,\n Character delim, Character escape, \n Character quote) \n throws IOException, ParseException {\n String retstring;\n if (null == delim) {\n return null;\n }\n if (!il.hasNext())\n return \"\";\n char c = il.getNext();\n if (c == delim) {\n retstring = \"\";\n }\n else {\n StringBuilder sb = new StringBuilder(10240).append(c);\n String s = extractUntil(il, delim, escape, quote, (c == quote));\n if (null == s) {\n return null;\n }\n retstring = sb.append(s).toString();\n }\n return prepareToParse(retstring, nullString, quote);\n }\n\n public String extractUntil(IndexedLine il, Character delim, \n Character escape, Character quote,\n boolean inquote)\n throws IOException, ParseException {\n if (null == delim) {\n return null;\n }\n StringBuilder sb = new StringBuilder(10240);\n char c;\n while (il.hasNext()) {\n c = il.getNext();\n if ((c == delim) && (!inquote)) {\n break;\n }\n sb.append(c);\n if (null != quote) {\n if (c == quote) {\n inquote = !inquote;\n }\n }\n if (null != escape) {\n if (c == escape) {\n c = il.getNext();\n sb.append(Character.toChars(c));\n }\n }\n }\n return sb.toString();\n }\n\n public static String quote(String instr) {\n if (null == instr)\n return null;\n return \"\\\"\" + escape(instr) + \"\\\"\";\n }\n\n public static String unquote(String instr) {\n if (null == instr)\n return null;\n if ((instr.startsWith(\"\\\"\")) && (instr.endsWith(\"\\\"\")))\n return unescape(instr.substring(1, instr.length() - 1));\n return instr;\n }\n\n public static String unstripQuote(String instr) {\n if (null == instr)\n return null;\n return \"\\\"\" + instr + \"\\\"\";\n }\n\n public static String stripQuote(String instr) {\n if (null == instr)\n return null;\n if ((instr.startsWith(\"\\\"\")) && (instr.endsWith(\"\\\"\")))\n return instr.substring(1, instr.length() - 1);\n return instr;\n }\n\n public static String escape(String instr) {\n if (null == instr)\n return null;\n return StringEscapeUtils.escapeJava(instr);\n }\n\n public static String unescape(String instr) {\n if (null == instr)\n return null;\n return StringEscapeUtils.unescapeJava(instr);\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/BigDecimalParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.math.BigDecimal;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// BigDecimal parser\npublic class BigDecimalParser extends AbstractParser {\n public BigDecimal parseIt(String toparse) throws NumberFormatException {\n if (null == toparse)\n return null;\n return new BigDecimal(toparse);\n }\n\n public String format(Object o) {\n BigDecimal v = (BigDecimal)o;\n return v.toString();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/BigIntegerParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.math.BigInteger;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// BigInteger parser\npublic class BigIntegerParser extends AbstractParser {\n public BigInteger parseIt(String toparse) throws NumberFormatException {\n if (null == toparse)\n return null;\n return new BigInteger(toparse);\n }\n\n public String format(Object o) {\n BigInteger v = (BigInteger)o;\n return v.toString();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/BooleanParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.text.ParseException;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Boolean parser - handles any way that Booleans can be expressed in Java\npublic class BooleanParser extends AbstractParser {\n public static enum BoolStyle { \n BoolStyle_TrueFalse(\"TRUE_FALSE\", \"TRUE\", \"FALSE\"), \n BoolStyle_10(\"1_0\", \"1\", \"0\"), \n BoolStyle_TF(\"T_F\", \"T\", \"F\"), \n BoolStyle_YN(\"Y_N\", \"Y\", \"N\"), \n BoolStyle_YesNo(\"YES_NO\", \"YES\", \"NO\");\n\n private String styleStr;\n private String trueStr;\n private String falseStr;\n\n BoolStyle(String inStyleStr, String inTrueStr, String inFalseStr) {\n styleStr = inStyleStr;\n trueStr = inTrueStr;\n falseStr = inFalseStr;\n }\n\n public String getStyle() {\n return styleStr;\n }\n\n public String getTrueStr() {\n return trueStr;\n }\n\n public String getFalseStr() {\n return falseStr;\n }\n }\n \n private String boolTrue;\n private String boolFalse;\n private static String BOOLSTYLE_1_0 = \"1_0\";\n private static String BOOLSTYLE_T_F = \"T_F\";\n private static String BOOLSTYLE_Y_N = \"Y_N\";\n private static String BOOLSTYLE_YES_NO = \"YES_NO\";\n private static String BOOLSTYLE_TRUE_FALSE = \"TRUE_FALSE\";\n \n public BooleanParser() {\n this(BoolStyle.BoolStyle_TrueFalse);\n }\n \n public BooleanParser(BoolStyle inBoolStyle) {\n if (null == inBoolStyle)\n inBoolStyle = BoolStyle.BoolStyle_TrueFalse;\n boolTrue = inBoolStyle.getTrueStr();\n boolFalse = inBoolStyle.getFalseStr();\n }\n\n public BooleanParser(String inBoolTrue, String inBoolFalse) {\n boolTrue = inBoolTrue;\n boolFalse = inBoolFalse; \n }\n\n public static BoolStyle getBoolStyle(String instr) {\n for (BoolStyle bs : BoolStyle.values()) {\n if (bs.getStyle().equalsIgnoreCase(instr)) {\n return bs;\n }\n }\n return null;\n }\n\n public static String getOptions() {\n String ret = \"'\" + BOOLSTYLE_1_0 + \"'\";\n ret = ret + \", '\" + BOOLSTYLE_T_F + \"'\";\n ret = ret + \", '\" + BOOLSTYLE_Y_N + \"'\";\n ret = ret + \", '\" + BOOLSTYLE_TRUE_FALSE + \"'\";\n ret = ret + \", '\" + BOOLSTYLE_YES_NO + \"'\";\n return ret;\n };\n\n public Boolean parseIt(String toparse) throws ParseException {\n if (null == toparse)\n return null;\n if (boolTrue.equalsIgnoreCase(toparse))\n return new Boolean(\"TRUE\");\n if (boolFalse.equalsIgnoreCase(toparse))\n return new Boolean(\"FALSE\");\n throw new ParseException(\"Boolean was not TRUE (\" + boolTrue + \") or FALSE (\" + boolFalse + \")\", 0);\n }\n\n public String format(Object o) {\n Boolean v = (Boolean)o;\n if (v)\n return boolTrue;\n return boolFalse;\n }\n}\n\n" }, { "path": "src/main/java/com/datastax/loader/parser/ByteBufferParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.nio.ByteBuffer;\nimport javax.xml.bind.DatatypeConverter;\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// ByteBuffer parser\npublic class ByteBufferParser extends AbstractParser {\n public ByteBuffer parseIt(String toparse) {\n if (null == toparse)\n return null;\n byte[] barry = DatatypeConverter.parseBase64Binary(toparse);\n return ByteBuffer.wrap(barry);\n }\n\n public String format(Object o) {\n ByteBuffer v = (ByteBuffer)o;\n return DatatypeConverter.printBase64Binary(v.array());\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/ByteParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Locale;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Byte parser - use the Number parser\npublic class ByteParser extends NumberParser {\n public ByteParser() {\n super();\n }\n \n public ByteParser(Locale inLocale) {\n super(inLocale);\n }\n \n public ByteParser(Locale inLocale, Boolean grouping) {\n super(inLocale, grouping);\n }\n \n public Byte parseIt(String toparse) throws ParseException {\n Number val = super.parseIt(toparse);\n return (null == val) ? null : val.byteValue();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/DateParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Date;\nimport java.util.Locale;\nimport java.text.DateFormat;\nimport java.text.SimpleDateFormat;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Date parser - takes a format string\npublic class DateParser extends AbstractParser {\n private DateFormat format;\n public DateParser(String inFormatString) {\n if (null == inFormatString)\n format = new SimpleDateFormat();\n else\n format = new SimpleDateFormat(inFormatString, Locale.ENGLISH);\n }\n \n public Date parseIt(String toparse) throws ParseException {\n if (null == toparse)\n return null;\n return format.parse(toparse);\n }\n\n public String format(Object o) {\n Date v = (Date)o;\n if (v == null)\n return null;\n return format.format(v);\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/DelimParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\nimport com.univocity.parsers.csv.CsvParser;\nimport com.univocity.parsers.csv.CsvParserSettings;\n\nimport java.io.IOException;\nimport java.text.ParseException;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class DelimParser {\n private List parsers;\n private int parsersSize;\n private List elements;\n private String delimiter;\n private int charsPerColumn;\n private String nullString;\n private char delim;\n private char quote;\n private char escape;\n private char comment;\n private List skip;\n\n private CsvParser csvp = null;\n\n public static String DEFAULT_DELIMITER = \",\";\n public static String DEFAULT_NULLSTRING = \"\";\n public static String DEFAULT_COMMENT_STRING = \"\\0\";\n public static int DEFAULT_CHARSPERCOLUMN = 4096;\n\n public DelimParser() {\n this(DEFAULT_DELIMITER);\n }\n\n public DelimParser(String inDelimiter) {\n this(inDelimiter, DEFAULT_CHARSPERCOLUMN, DEFAULT_NULLSTRING, DEFAULT_COMMENT_STRING);\n }\n\n public DelimParser(String inDelimiter, int inCharsPerColumn,\n String inNullString, String inComment) {\n parsers = new ArrayList();\n elements = new ArrayList();\n skip = new ArrayList();\n parsersSize = parsers.size();\n if (null == inDelimiter)\n delimiter = DEFAULT_DELIMITER;\n else\n delimiter = inDelimiter;\n if (null == inNullString)\n nullString = DEFAULT_NULLSTRING;\n else\n nullString = inNullString;\n if (null == inComment)\n comment = DEFAULT_COMMENT_STRING.charAt(0);\n else\n comment = inComment.charAt(0);\n charsPerColumn = inCharsPerColumn;\n delim = (\"\\\\t\".equals(delimiter)) ? '\\t' : delimiter.charAt(0);\n quote = '\\\"';\n escape = '\\\\';\n\n CsvParserSettings settings = new CsvParserSettings();\n settings.getFormat().setLineSeparator(\"\\n\");\n settings.getFormat().setDelimiter(delim);\n settings.setMaxCharsPerColumn(charsPerColumn);\n settings.getFormat().setQuote(quote);\n settings.getFormat().setQuoteEscape(escape);\n settings.getFormat().setCharToEscapeQuoteEscaping(escape);\n settings.setKeepQuotes(true);\n settings.setKeepEscapeSequences(true);\n settings.getFormat().setComment(comment);\n\n csvp = new CsvParser(settings);\n }\n\n // Adds a parser to the list\n public void add(Parser p) {\n parsers.add(p);\n skip.add(false);\n parsersSize = parsers.size();\n }\n\n public void addSkip(int idx) {\n parsers.add(idx, new StringParser());\n skip.add(idx, true);\n parsersSize = parsers.size();\n }\n\n // This is where we apply rules like quoting, NULL, etc\n private String prepareToParse(String toparse) {\n String trimmedToParse = toparse.trim();\n if (trimmedToParse.startsWith(\"\\\"\") && trimmedToParse.endsWith(\"\\\"\"))\n trimmedToParse = trimmedToParse.substring(1, trimmedToParse.length() - 1);\n if (trimmedToParse.equals(nullString))\n return null;\n return trimmedToParse;\n }\n\n public List parse(String line) {\n //return parseComplex(line);\n return parseWithUnivocity(line);\n }\n\n public List parseWithUnivocity(String line) {\n String[] row = csvp.parseLine(line);\n return parse(row);\n }\n\n public List parse(String[] row) {\n if (row.length != parsersSize) {\n System.err.println(\"Row has different number of fields (\" + row.length + \") than expected (\" + parsersSize + \")\");\n return null;\n }\n elements.clear();\n Object toAdd;\n for (int i = 0; i < parsersSize; i++) {\n try {\n String toparse = row[i];\n if ((null == toparse) ||\n ((null != nullString) &&\n (nullString.equalsIgnoreCase(AbstractParser.unquote(toparse)))))\n toAdd = null;\n else\n toAdd = parsers.get(i).parse(toparse);\n\n if (!skip.get(i))\n elements.add(toAdd);\n }\n catch (NumberFormatException e) {\n System.err.println(String.format(\"Invalid number in input number %d: %s\", i, e.getMessage()));\n return null;\n }\n catch (ParseException pe) {\n System.err.println(String.format(\"Invalid format in input %d: %s\", i, pe.getMessage()));\n return null;\n }\n }\n\n return elements;\n }\n\n public List parseComplex(String line) {\n elements.clear();\n IndexedLine sr = new IndexedLine(line);\n for (int i = 0; i < parsersSize; i++) {\n try {\n Object toAdd = parsers.get(i).parse(sr, nullString, delim, \n escape, quote, \n (parsersSize-1 == i));\n if (!skip.get(i))\n elements.add(toAdd);\n }\n catch (NumberFormatException e) {\n System.err.println(String.format(\"Invalid number in input number %d: %s\", i, e.getMessage()));\n return null;\n }\n catch (ParseException pe) {\n System.err.println(String.format(\"Invalid format in input %d: %s\", i, pe.getMessage()));\n return null;\n }\n catch (IOException e) {\n System.err.println(String.format(\"Invalid number of fields - ran out of string: %s\", i, e.getMessage()));\n return null;\n }\n }\n return elements;\n }\n\n // returns an array of Objects - to be used in PreparedStatement.bind()\n public Object[] getElements() {\n return elements.toArray();\n }\n\n public String format(Row row) throws IndexOutOfBoundsException, InvalidTypeException {\n StringBuilder retVal = new StringBuilder();\n String[] stringVals = stringVals(row);\n retVal.append(stringVals[0]);\n for (int i = 1; i < parsersSize; i++) {\n retVal.append(delimiter).append(stringVals[i]);\n }\n return retVal.toString();\n }\n\n public String[] stringVals(Row row) throws IndexOutOfBoundsException, InvalidTypeException {\n String[] stringVals = new String[parsers.size()];\n for (int i = 0; i < parsersSize; i++) {\n stringVals[i] = parsers.get(i).format(row, i);\n if (null == stringVals[i])\n stringVals[i] = nullString;\n }\n return stringVals;\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/DoubleParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Locale;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Double parser - use the Number parser\npublic class DoubleParser extends NumberParser {\n public DoubleParser() {\n super();\n }\n \n public DoubleParser(Locale inLocale) {\n super(inLocale);\n }\n\n public DoubleParser(Locale inLocale, Boolean grouping) {\n super(inLocale, grouping);\n }\n \n public Double parseIt(String toparse) throws ParseException {\n Number val = super.parseIt(toparse);\n return (null == val) ? null : val.doubleValue();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/FloatParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Locale;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Float parser - use the Number parser\npublic class FloatParser extends NumberParser {\n public FloatParser() {\n super();\n }\n \n public FloatParser(Locale inLocale) {\n super(inLocale);\n }\n \n public FloatParser(Locale inLocale, Boolean grouping) {\n super(inLocale, grouping);\n }\n\n public Float parseIt(String toparse) throws ParseException {\n Number val = super.parseIt(toparse);\n return (null == val) ? null : val.floatValue();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/IndexedLine.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.text.ParseException;\n\npublic class IndexedLine {\n private int index;\n private char[] buffer;\n\n public IndexedLine(char[] inBuffer) {\n buffer = inBuffer;\n index = 0;\n }\n\n public IndexedLine(String instr) {\n this(instr.toCharArray());\n }\n\n public char getNext() throws ParseException {\n if (buffer.length == index)\n throw new ParseException(\"ran out of buffer (\" + index + \" / \" + buffer.length + \")\", 0);\n char c = buffer[index];\n index++;\n return c;\n }\n\n public boolean hasNext() {\n return buffer.length > index;\n }\n\n public boolean setIndex(int idx) throws ParseException {\n if (buffer.length - 1 < idx)\n throw new ParseException(\"index out of range\", 0);\n index = idx;\n return true;\n }\n\n public char get(int idx) throws ParseException {\n if (buffer.length - 1 < idx)\n throw new ParseException(\"index out of range\", 0);\n return buffer[idx];\n }\n\n public boolean backup() {\n if (0 == index)\n return false;\n index--;\n return true;\n }\n\n public int length() {\n return buffer.length;\n }\n\n public int numRemaining() {\n return buffer.length - index;\n }\n\n public String remaining() {\n return new String(buffer, index, numRemaining());\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/InetAddressParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.net.InetAddress;\nimport java.text.ParseException;\nimport java.net.UnknownHostException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// InetAddress parser\npublic class InetAddressParser extends AbstractParser {\n public InetAddress parseIt(String toparse) throws ParseException {\n if (null == toparse)\n return null;\n InetAddress ret;\n try {\n ret = InetAddress.getByName(toparse);\n }\n catch (UnknownHostException uhe) {\n throw new ParseException(\"Error parsing Inet: \" + uhe.getMessage(), 0);\n }\n return ret;\n }\n\n public String format(Object o) {\n InetAddress v = (InetAddress)o;\n return v.getHostAddress();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/IntegerParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Locale;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Integer parser - use the Number parser\npublic class IntegerParser extends NumberParser {\n public IntegerParser() {\n super();\n }\n \n public IntegerParser(Locale inLocale) {\n super(inLocale);\n }\n \n public IntegerParser(Locale inLocale, Boolean grouping) {\n super(inLocale, grouping);\n }\n \n public Integer parseIt(String toparse) throws ParseException {\n Number val = super.parseIt(toparse);\n return (null == val) ? null : val.intValue();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/ListParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.List;\nimport java.util.ArrayList;\nimport java.io.StringReader;\nimport java.io.IOException;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\nimport com.univocity.parsers.csv.CsvParser;\nimport com.univocity.parsers.csv.CsvParserSettings;\n\npublic class ListParser extends AbstractParser {\n private Parser parser;\n private char collectionDelim;\n private char collectionBegin;\n private char collectionEnd;\n private char collectionQuote = '\\\"';\n private char collectionEscape = '\\\\';\n private String collectionNullString = \"null\";\n private List elements;\n \n private CsvParser csvp = null;\n \n public ListParser(Parser inParser, char inCollectionDelim, \n char inCollectionBegin, char inCollectionEnd) {\n parser = inParser;\n collectionDelim = inCollectionDelim;\n collectionBegin = inCollectionBegin;\n collectionEnd = inCollectionEnd;\n elements = new ArrayList();\n\n CsvParserSettings settings = new CsvParserSettings();\n settings.getFormat().setLineSeparator(\"\\n\");\n settings.getFormat().setDelimiter(collectionDelim);\n settings.getFormat().setQuote(collectionQuote);\n settings.getFormat().setQuoteEscape(collectionEscape);\n settings.getFormat().setCharToEscapeQuoteEscaping(collectionEscape);\n settings.setKeepQuotes(true);\n settings.setKeepEscapeSequences(true);\n \n csvp = new CsvParser(settings);\n }\n\n public Object parseIt(String toparse) throws ParseException {\n if (null == toparse)\n return null;\n if (!toparse.startsWith(Character.toString(collectionBegin)))\n throw new ParseException(\"Must begin with \" + collectionBegin \n + \"\\n\", 0);\n if (!toparse.endsWith(Character.toString(collectionEnd)))\n throw new ParseException(\"Must end with \" + collectionEnd \n + \"\\n\", 0);\n toparse = toparse.substring(1, toparse.length() - 1);\n String[] row = csvp.parseLine(toparse);\n elements.clear();\n try {\n for (int i = 0; i < row.length; i++) \n elements.add(parser.parse(row[i]));\n }\n catch (Exception e) {\n System.err.println(\"Trouble parsing : \" + e.getMessage());\n return null;\n }\n return elements;\n }\n\n //public String format(Row row, int index) {\n // if (row.isNull(index))\n // return null;\n // List list = row.getList(index, Object.class);\n @SuppressWarnings(\"unchecked\")\n public String format(Object o) {\n List list = (List)o;\n StringBuilder sb = new StringBuilder();\n sb.append(collectionBegin);\n if (list.size() > 0) {\n for (int i = 0; i < list.size() - 1; i++) {\n sb.append(parser.format(list.get(i)));\n sb.append(collectionDelim);\n }\n sb.append(parser.format(list.get(list.size() - 1)));\n }\n sb.append(collectionEnd);\n return quote(sb.toString());\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/LocalDateParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Date;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.LocalDate;\n\npublic class LocalDateParser extends AbstractParser {\n private DateParser dateParser;\n\n public LocalDateParser(String inFormatString) {\n dateParser = new DateParser(inFormatString);\n }\n\n public LocalDate parseIt(String toparse) throws ParseException {\n if (null == toparse)\n return null;\n Date d = dateParser.parseIt(toparse);\n LocalDate ret = LocalDate.fromMillisSinceEpoch(d.getTime());\n return ret;\n }\n\n public String format(Object o) {\n LocalDate v = (LocalDate)o;\n if (v == null)\n return null;\n Date d = new Date(v.getMillisSinceEpoch());\n return dateParser.format(d);\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/LongParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Locale;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Long parser - use the Number parser\npublic class LongParser extends NumberParser {\n public LongParser() {\n super();\n }\n \n public LongParser(Locale inLocale) {\n super(inLocale);\n }\n \n public LongParser(Locale inLocale, Boolean grouping) {\n super(inLocale, grouping);\n }\n \n public Long parseIt(String toparse) throws ParseException {\n Number val = super.parseIt(toparse);\n return (null == val) ? null : val.longValue();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/MapParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.io.StringReader;\nimport java.io.IOException;\nimport java.io.StringReader;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\nimport com.univocity.parsers.csv.CsvParser;\nimport com.univocity.parsers.csv.CsvParserSettings;\n\npublic class MapParser extends AbstractParser {\n private Parser keyParser;\n private Parser valueParser;\n private char collectionDelim;\n private char collectionBegin;\n private char collectionEnd;\n private char collectionQuote = '\\\"';\n private char collectionEscape = '\\\\';\n private char mapDelim;\n private String collectionNullString = null;\n private Map elements;\n \n private CsvParser csvp = null;\n\n public MapParser(Parser inKeyParser, Parser inValueParser,\n char inCollectionDelim, char inCollectionBegin, \n char inCollectionEnd, char inMapDelim) {\n keyParser = inKeyParser;\n valueParser = inValueParser;\n collectionDelim = inCollectionDelim;\n collectionBegin = inCollectionBegin;\n collectionEnd = inCollectionEnd;\n mapDelim = inMapDelim;\n elements = new HashMap();\n\n CsvParserSettings settings = new CsvParserSettings();\n settings.getFormat().setLineSeparator(\"\" + collectionDelim);\n settings.getFormat().setNormalizedNewline(collectionDelim);\n settings.getFormat().setDelimiter(mapDelim);\n settings.getFormat().setQuote(collectionQuote);\n settings.getFormat().setQuoteEscape(collectionEscape);\n settings.getFormat().setCharToEscapeQuoteEscaping(collectionEscape);\n settings.setKeepQuotes(true);\n settings.setKeepEscapeSequences(true);\n \n csvp = new CsvParser(settings);\n }\n public Object parseIt(String toparse) throws ParseException {\n if (null == toparse)\n return null;\n if (!toparse.startsWith(Character.toString(collectionBegin)))\n throw new ParseException(\"Must begin with \" + collectionBegin \n + \"\\n\", 0);\n if (!toparse.endsWith(Character.toString(collectionEnd)))\n throw new ParseException(\"Must end with \" + collectionEnd \n + \"\\n\", 0);\n toparse = toparse.substring(1, toparse.length() - 1);\n elements.clear();\n StringReader sr = new StringReader(toparse);\n csvp.beginParsing(sr);\n try {\n String[] row;\n while ((row = csvp.parseNext()) != null) {\n Object key = keyParser.parse(row[0]);\n Object value = valueParser.parse(row[1]);\n elements.put(key, value);\n }\n }\n catch (Exception e) {\n System.err.println(\"Trouble parsing : \" + e.getMessage());\n e.printStackTrace();\n return null;\n }\n return elements;\n }\n\n @SuppressWarnings(\"unchecked\")\n public String format(Object o) {\n Map map = (Map)o;\n Iterator > iter = map.entrySet().iterator();\n Map.Entry me;\n StringBuilder sb = new StringBuilder();\n sb.append(collectionBegin);\n if (iter.hasNext()) {\n me = iter.next();\n sb.append(keyParser.format(me.getKey()));\n sb.append(mapDelim);\n sb.append(valueParser.format(me.getValue()));\n }\n while (iter.hasNext()) {\n sb.append(collectionDelim);\n me = iter.next();\n sb.append(keyParser.format(me.getKey()));\n sb.append(mapDelim);\n sb.append(valueParser.format(me.getValue()));\n }\n sb.append(collectionEnd);\n\n return quote(sb.toString());\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/NumberParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Locale;\nimport java.text.NumberFormat;\nimport java.text.DecimalFormat;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// General number parser\n// This is useful as it can take care of Locales for us\n// That means comma as a decimal separator, etc.\npublic class NumberParser extends AbstractParser {\n protected NumberFormat nf;\n public NumberParser() {\n this(null);\n }\n \n public NumberParser(Locale locale) {\n this(locale, true);\n }\n\n public NumberParser(Locale locale, Boolean grouping) {\n if (null == locale)\n locale = Locale.ENGLISH;\n nf = NumberFormat.getInstance(locale);\n if (nf instanceof DecimalFormat) {\n ((DecimalFormat) nf).setGroupingUsed(grouping);\n }\n }\n \n // Need this method for the subclasses\n public Number parseIt(String toparse) throws ParseException {\n if ((null == toparse) || (0 == toparse.length()))\n return null;\n return nf.parse(toparse);\n }\n\n public String format(Object o) {\n return nf.format(o);\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/Parser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.text.ParseException;\nimport java.io.StringReader;\nimport java.io.IOException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Parsing Interface - one method parse(String)\npublic interface Parser {\n public Object parse(String toparse) throws ParseException;\n public Object parse(IndexedLine il, String nullString, Character delim, \n Character escape, Character quote, boolean last)\n throws IOException, ParseException;\n public String format(Row row, int index) throws IndexOutOfBoundsException, InvalidTypeException;\n public String format(Object o);\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/SetParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Set;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.io.StringReader;\nimport java.io.IOException;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\nimport com.univocity.parsers.csv.CsvParser;\nimport com.univocity.parsers.csv.CsvParserSettings;\n\npublic class SetParser extends AbstractParser {\n private Parser parser;\n private char collectionDelim;\n private char collectionBegin;\n private char collectionEnd;\n private char collectionQuote = '\\\"';\n private char collectionEscape = '\\\\';\n private String collectionNullString = \"null\";\n private Set elements;\n \n private CsvParser csvp = null;\n\n public SetParser(Parser inParser, char inCollectionDelim, \n char inCollectionBegin, char inCollectionEnd) {\n parser = inParser;\n collectionDelim = inCollectionDelim;\n collectionBegin = inCollectionBegin;\n collectionEnd = inCollectionEnd;\n elements = new HashSet();\n\n CsvParserSettings settings = new CsvParserSettings();\n settings.getFormat().setLineSeparator(\"\\n\");\n settings.getFormat().setDelimiter(collectionDelim);\n settings.getFormat().setQuote(collectionQuote);\n settings.getFormat().setQuoteEscape(collectionEscape);\n settings.getFormat().setCharToEscapeQuoteEscaping(collectionEscape);\n settings.setKeepQuotes(true);\n settings.setKeepEscapeSequences(true);\n \n csvp = new CsvParser(settings);\n }\n public Object parseIt(String toparse) throws ParseException {\n if (null == toparse)\n return null;\n if (!toparse.startsWith(Character.toString(collectionBegin)))\n throw new ParseException(\"Must begin with \" + collectionBegin \n + \"\\n\", 0);\n if (!toparse.endsWith(Character.toString(collectionEnd)))\n throw new ParseException(\"Must end with \" + collectionEnd \n + \"\\n\", 0);\n toparse = toparse.substring(1, toparse.length() - 1);\n String[] row = csvp.parseLine(toparse);\n elements.clear();\n try {\n for (int i = 0; i < row.length; i++) \n elements.add(parser.parse(row[i]));\n }\n catch (Exception e) {\n System.err.println(\"Trouble parsing : \" + e.getMessage());\n e.printStackTrace();\n return null;\n }\n return elements;\n }\n\n //public String format(Row row, int index) {\n // if (row.isNull(index))\n // return null;\n // Set set = row.getSet(index, Object.class);\n @SuppressWarnings(\"unchecked\")\n public String format(Object o) {\n Set set = (Set)o;\n Iterator iter = set.iterator();\n StringBuilder sb = new StringBuilder();\n sb.append(collectionBegin);\n if (iter.hasNext())\n sb.append(parser.format(iter.next()));\n while (iter.hasNext()) {\n sb.append(collectionDelim);\n sb.append(parser.format(iter.next()));\n }\n sb.append(collectionEnd);\n\n return quote(sb.toString());\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/ShortParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.Locale;\nimport java.text.ParseException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// Short parser - use the Number parser\npublic class ShortParser extends NumberParser {\n public ShortParser() {\n super();\n }\n \n public ShortParser(Locale inLocale) {\n super(inLocale);\n }\n \n public ShortParser(Locale inLocale, Boolean grouping) {\n super(inLocale, grouping);\n }\n \n public Short parseIt(String toparse) throws ParseException {\n Number val = super.parseIt(toparse);\n return (null == val) ? null : val.shortValue();\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/StringParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.io.StringReader;\nimport java.io.IOException;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// String parser - simple\npublic class StringParser extends AbstractParser {\n public String parseIt(String toparse) {\n return toparse;\n }\n\n public String format(Object o) {\n String iv = (String)o;\n return quote(iv);\n }\n}\n" }, { "path": "src/main/java/com/datastax/loader/parser/UUIDParser.java", "content": "/*\n * Copyright 2015 Brian Hess\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.datastax.loader.parser;\n\nimport java.util.UUID;\n\nimport com.datastax.driver.core.Row;\nimport com.datastax.driver.core.exceptions.InvalidTypeException;\n\n// UUID parser\npublic class UUIDParser extends AbstractParser {\n public UUID parseIt(String toparse) throws IllegalArgumentException {\n if (null == toparse)\n return null;\n return UUID.fromString(toparse);\n }\n public String format(Object o) {\n UUID v = (UUID)o;\n return v.toString();\n }\n}\n" }, { "path": "src/main/resources/logback.xml", "content": "\n\n \n   \n   %-5p %msg%n\n \n  \n \n   \n  \n\n" }, { "path": "src/make/buildit.sh", "content": "#!/bin/sh\n\ncat src/make/cassandra-loader.sh build/libs/cassandra-loader-uber*.jar > build/cassandra-loader && chmod 755 build/cassandra-loader\n" }, { "path": "src/make/cassandra-loader.sh", "content": "#!/bin/sh\nMYSELF=`which \"$0\" 2>/dev/null`\n[ $? -gt 0 -a -f \"$0\" ] && MYSELF=\"./$0\"\njava=java\nif test -n \"$JAVA_HOME\"; then\n java=\"$JAVA_HOME/bin/java\"\nfi\nexec \"$java\" -XX:+UseG1GC -Xmx1G -Xms1G -XX:+UseTLAB -XX:+ResizeTLAB $java_args -jar $MYSELF \"$@\"\nexit 1 \n" }, { "path": "src/make/unloader.sh", "content": "#!/bin/sh\n\ncat src/make/cassandra-loader.sh build/libs/cassandra-unloader-uber*.jar > build/cassandra-unloader && chmod 755 build/cassandra-unloader\n" } ]