Repository: jcustenborder/kafka-connect-spooldir Branch: master Commit: 7506b3da0701 Files: 140 Total size: 2.1 MB Directory structure: gitextract_f2vujxj0/ ├── .gitignore ├── Jenkinsfile ├── LICENSE ├── README.md ├── bin/ │ └── debug.sh ├── config/ │ ├── AvroExample.properties │ ├── CSVExample.json │ ├── CSVExample.properties │ ├── CSVSchemaGenerator.properties │ ├── JsonExample.properties │ └── connect-avro-docker.properties ├── docker-compose.yml ├── pom.xml └── src/ ├── main/ │ └── java/ │ └── com/ │ └── github/ │ └── jcustenborder/ │ └── kafka/ │ └── connect/ │ └── spooldir/ │ ├── AbstractCleanUpPolicy.java │ ├── AbstractSchemaGenerator.java │ ├── AbstractSourceConnector.java │ ├── AbstractSourceConnectorConfig.java │ ├── AbstractSourceTask.java │ ├── AbstractSpoolDirSourceConnector.java │ ├── AbstractSpoolDirSourceConnectorConfig.java │ ├── AbstractSpoolDirSourceTask.java │ ├── AbstractTaskPartitionerPredicate.java │ ├── CsvSchemaGenerator.java │ ├── FileComparator.java │ ├── InputFile.java │ ├── InputFileDequeue.java │ ├── JsonSchemaGenerator.java │ ├── Metadata.java │ ├── SpoolDirAvroSourceConnector.java │ ├── SpoolDirAvroSourceConnectorConfig.java │ ├── SpoolDirAvroSourceTask.java │ ├── SpoolDirBinaryFileSourceConnector.java │ ├── SpoolDirBinaryFileSourceConnectorConfig.java │ ├── SpoolDirBinaryFileSourceTask.java │ ├── SpoolDirCsvSourceConnector.java │ ├── SpoolDirCsvSourceConnectorConfig.java │ ├── SpoolDirCsvSourceTask.java │ ├── SpoolDirJsonSourceConnector.java │ ├── SpoolDirJsonSourceConnectorConfig.java │ ├── SpoolDirJsonSourceTask.java │ ├── SpoolDirLineDelimitedSourceConnector.java │ ├── SpoolDirLineDelimitedSourceConnectorConfig.java │ ├── SpoolDirLineDelimitedSourceTask.java │ ├── SpoolDirSchemaLessJsonSourceConnector.java │ ├── SpoolDirSchemaLessJsonSourceConnectorConfig.java │ ├── SpoolDirSchemaLessJsonSourceTask.java │ ├── elf/ │ │ ├── SchemaConversion.java │ │ ├── SchemaConversionBuilder.java │ │ ├── SpoolDirELFSourceConnector.java │ │ ├── SpoolDirELFSourceConnectorConfig.java │ │ ├── SpoolDirELFSourceTask.java │ │ └── converters/ │ │ ├── LocalDateLogFieldConverter.java │ │ ├── LocalTimeLogFieldConverter.java │ │ ├── LogFieldConverter.java │ │ ├── LogFieldConverterFactory.java │ │ ├── PrimitiveLogFieldConverter.java │ │ └── TimestampLogFieldConverter.java │ └── package-info.java └── test/ ├── java/ │ └── com/ │ └── github/ │ └── jcustenborder/ │ └── kafka/ │ └── connect/ │ └── spooldir/ │ ├── AbstractCleanUpPolicyTest.java │ ├── AbstractSchemaGeneratorTest.java │ ├── AbstractSpoolDirSourceConnectorTest.java │ ├── AbstractSpoolDirSourceTaskTest.java │ ├── ByNameAbstractTaskPartitionerPredicateTest.java │ ├── CsvSchemaGeneratorTest.java │ ├── DeleteCleanupPolicySubDirsNoRetainTest.java │ ├── DeleteCleanupPolicySubDirsRetainTest.java │ ├── DeleteCleanupPolicyTest.java │ ├── DocumentationTest.java │ ├── FileComparatorTest.java │ ├── JsonSchemaGeneratorTest.java │ ├── MinimumFileAgePredicateTest.java │ ├── MoveByDateCleanupPolicySubDirsNoRetainTest.java │ ├── MoveByDateCleanupPolicySubDirsRetainTest.java │ ├── MoveByDateCleanupPolicyTest.java │ ├── MoveCleanupPolicySubDirsNoRetainTest.java │ ├── MoveCleanupPolicySubDirsRetainTest.java │ ├── MoveCleanupPolicyTest.java │ ├── NamedTest.java │ ├── NoneCleanupPolicyTest.java │ ├── ProcessingFileExistsPredicateTest.java │ ├── SpoolDirAvroSourceTaskTest.java │ ├── SpoolDirBinaryFileSourceTaskTest.java │ ├── SpoolDirCsvSourceConnectorConfigTest.java │ ├── SpoolDirCsvSourceConnectorTest.java │ ├── SpoolDirCsvSourceTaskSubDirsNoRetainTest.java │ ├── SpoolDirCsvSourceTaskSubDirsRetainTest.java │ ├── SpoolDirCsvSourceTaskTest.java │ ├── SpoolDirJsonSourceConnectorTest.java │ ├── SpoolDirJsonSourceTaskTest.java │ ├── SpoolDirLineDelimitedSourceTaskTest.java │ ├── SpoolDirSchemaLessJsonSourceTaskTest.java │ ├── TestCase.java │ ├── TestDataUtils.java │ └── elf/ │ ├── SchemaConversionBuilderTest.java │ └── SpoolDirELFSourceTaskTest.java └── resources/ ├── com/ │ └── github/ │ └── jcustenborder/ │ └── kafka/ │ └── connect/ │ └── spooldir/ │ ├── SpoolBinaryFileSourceConnector/ │ │ ├── binary.json │ │ └── fromXML.json │ ├── SpoolDirBinaryFileSourceConnector/ │ │ ├── binary.json │ │ └── fromXML.json │ ├── SpoolDirCsvSourceConnector/ │ │ ├── schema.json │ │ ├── schemaheaders.json │ │ └── tsv.json │ ├── SpoolDirJsonSourceConnector/ │ │ └── test.json │ ├── SpoolDirLineDelimitedSourceConnector/ │ │ └── fix.json │ ├── avro/ │ │ ├── FieldsMatch.data │ │ └── FieldsMatch.json │ ├── binary/ │ │ ├── DataHasMoreFields.data │ │ └── DataHasMoreFields.json │ ├── csv/ │ │ ├── BlankLines.data │ │ ├── BlankLines.json │ │ ├── DataHasMoreFields.data │ │ ├── DataHasMoreFields.json │ │ ├── FieldsMatch.data │ │ ├── FieldsMatch.json │ │ ├── FileModeFieldFieldsMatch.data │ │ ├── FileModeFieldFieldsMatch.json │ │ ├── SchemaHasMoreFields.data │ │ ├── SchemaHasMoreFields.json │ │ ├── SourceOffset.data │ │ ├── SourceOffset.json │ │ ├── WithHeaderSkipLines.data │ │ ├── WithHeaderSkipLines.json │ │ ├── WithoutHeader.data │ │ └── WithoutHeader.json │ ├── elf/ │ │ ├── SpoolDirELFSourceConnector/ │ │ │ └── example.json │ │ └── elf/ │ │ ├── FieldsMatch.data │ │ └── FieldsMatch.json │ ├── json/ │ │ ├── DataHasMoreFields.data │ │ ├── DataHasMoreFields.json │ │ ├── FieldsMatch.data │ │ ├── FieldsMatch.json │ │ ├── FileModeFieldFieldsMatch.data │ │ ├── FileModeFieldFieldsMatch.json │ │ ├── SchemaHasMoreFields.data │ │ ├── SchemaHasMoreFields.json │ │ ├── SourceOffset.data │ │ └── SourceOffset.json │ └── schemalessjson/ │ ├── DataHasMoreFields.data │ └── DataHasMoreFields.json └── logback.xml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ target *.iml .okhttpcache ELFTesting.properties .checkstyle .factorypath .idea/ ================================================ FILE: Jenkinsfile ================================================ #!groovy @Library('jenkins-pipeline') import com.github.jcustenborder.jenkins.pipeline.KafkaConnectPipeline def pipe = new KafkaConnectPipeline() pipe.execute() ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # Introduction [Documentation](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir) | [Confluent Hub](https://www.confluent.io/hub/jcustenborder/kafka-connect-spooldir) This Kafka Connect connector provides the capability to watch a directory for files and read the data as new files are written to the input directory. Each of the records in the input file will be converted based on the user supplied schema. The connectors in this project handle all different kinds of use cases like ingesting json, csv, tsv, avro, or binary files. # Installation ## Confluent Hub The following command can be used to install the plugin directly from the Confluent Hub using the [Confluent Hub Client](https://docs.confluent.io/current/connect/managing/confluent-hub/client.html). ```bash confluent-hub install jcustenborder/kafka-connect-spooldir:latest ``` ## Manually The zip file that is deployed to the [Confluent Hub](https://www.confluent.io/hub/jcustenborder/kafka-connect-spooldir) is available under `target/components/packages/`. You can manually extract this zip file which includes all dependencies. All the dependencies that are required to deploy the plugin are under `target/kafka-connect-target` as well. Make sure that you include all the dependencies that are required to run the plugin. 1. Create a directory under the `plugin.path` on your Connect worker. 2. Copy all of the dependencies under the newly created subdirectory. 3. Restart the Connect worker. # Source Connectors ## [Schema Less Json Source Connector](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir/sources/SpoolDirSchemaLessJsonSourceConnector.html) ``` com.github.jcustenborder.kafka.connect.spooldir.SpoolDirSchemaLessJsonSourceConnector ``` This connector is used to `stream _` JSON files from a directory while converting the data based on the schema supplied in the configuration. This connector will read each file node by node writing the result to Kafka. For example if your data file contains several json objects the connector will read from { to } for each object and write each object to Kafka. ### Important This connector does not try to convert the json records to a schema. The recommended converter to use is the StringConverter. Example: `value.converter=org.apache.kafka.connect.storage.StringConverter` ### Configuration #### File System ##### `error.path` The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `input.file.pattern` Regular expression to check input file names against. This expression must match the entire filename. The equivalent of Matcher.matches(). *Importance:* HIGH *Type:* STRING ##### `input.path` The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `finished.path` The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING ##### `halt.on.error` Should the task halt when it encounters an error or continue to the next file. *Importance:* HIGH *Type:* BOOLEAN *Default Value:* true ##### `cleanup.policy` Determines how the connector should cleanup the files that have been successfully processed. NONE leaves the files in place which could cause them to be reprocessed if the connector is restarted. DELETE removes the file from the filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to a finished directory with subdirectories by date *Importance:* MEDIUM *Type:* STRING *Default Value:* MOVE *Validator:* Matches: ``NONE``, ``DELETE``, ``MOVE``, ``MOVEBYDATE`` ##### `task.partitioner` The task partitioner implementation is used when the connector is configured to use more than one task. This is used by each task to identify which files will be processed by that task. This ensures that each file is only assigned to one task. *Importance:* MEDIUM *Type:* STRING *Default Value:* ByName *Validator:* Matches: ``ByName`` ##### `file.buffer.size.bytes` The size of buffer for the BufferedInputStream that will be used to interact with the file system. *Importance:* LOW *Type:* INT *Default Value:* 131072 *Validator:* [1,...] ##### `file.minimum.age.ms` The amount of time in milliseconds after the file was last written to before the file can be processed. *Importance:* LOW *Type:* LONG *Default Value:* 0 *Validator:* [0,...] ##### `files.sort.attributes` The attributes each file will use to determine the sort order. `Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is the LastModified attribute of the file preferring older files first. *Importance:* LOW *Type:* LIST *Default Value:* [NameAsc] *Validator:* Matches: ``NameAsc``, ``NameDesc``, ``LengthAsc``, ``LengthDesc``, ``LastModifiedAsc``, ``LastModifiedDesc`` ##### `processing.file.extension` Before a file is processed, a flag is created in its directory to indicate the file is being handled. The flag file has the same name as the file, but with this property appended as a suffix. *Importance:* LOW *Type:* STRING *Default Value:* .PROCESSING *Validator:* Matches regex( ^.*\..+$ ) #### General ##### `topic` The Kafka topic to write the data to. *Importance:* HIGH *Type:* STRING ##### `batch.size` The number of records that should be returned with each batch. *Importance:* LOW *Type:* INT *Default Value:* 1000 ##### `empty.poll.wait.ms` The amount of time to wait if a poll returns an empty list of records. *Importance:* LOW *Type:* LONG *Default Value:* 500 *Validator:* [1,...,9223372036854775807] ##### `file.charset` Character set to read wth file with. *Importance:* LOW *Type:* STRING *Default Value:* UTF-8 *Validator:* Big5,Big5-HKSCS,CESU-8,EUC-JP,EUC-KR,GB18030,GB2312,GBK,IBM-Thai,IBM00858,IBM01140,IBM01141,IBM01142,IBM01143,IBM01144,IBM01145,IBM01146,IBM01147,IBM01148,IBM01149,IBM037,IBM1026,IBM1047,IBM273,IBM277,IBM278,IBM280,IBM284,IBM285,IBM290,IBM297,IBM420,IBM424,IBM437,IBM500,IBM775,IBM850,IBM852,IBM855,IBM857,IBM860,IBM861,IBM862,IBM863,IBM864,IBM865,IBM866,IBM868,IBM869,IBM870,IBM871,IBM918,ISO-2022-CN,ISO-2022-JP,ISO-2022-JP-2,ISO-2022-KR,ISO-8859-1,ISO-8859-13,ISO-8859-15,ISO-8859-16,ISO-8859-2,ISO-8859-3,ISO-8859-4,ISO-8859-5,ISO-8859-6,ISO-8859-7,ISO-8859-8,ISO-8859-9,JIS_X0201,JIS_X0212-1990,KOI8-R,KOI8-U,Shift_JIS,TIS-620,US-ASCII,UTF-16,UTF-16BE,UTF-16LE,UTF-32,UTF-32BE,UTF-32LE,UTF-8,windows-1250,windows-1251,windows-1252,windows-1253,windows-1254,windows-1255,windows-1256,windows-1257,windows-1258,windows-31j,x-Big5-HKSCS-2001,x-Big5-Solaris,x-euc-jp-linux,x-EUC-TW,x-eucJP-Open,x-IBM1006,x-IBM1025,x-IBM1046,x-IBM1097,x-IBM1098,x-IBM1112,x-IBM1122,x-IBM1123,x-IBM1124,x-IBM1129,x-IBM1166,x-IBM1364,x-IBM1381,x-IBM1383,x-IBM29626C,x-IBM300,x-IBM33722,x-IBM737,x-IBM833,x-IBM834,x-IBM856,x-IBM874,x-IBM875,x-IBM921,x-IBM922,x-IBM930,x-IBM933,x-IBM935,x-IBM937,x-IBM939,x-IBM942,x-IBM942C,x-IBM943,x-IBM943C,x-IBM948,x-IBM949,x-IBM949C,x-IBM950,x-IBM964,x-IBM970,x-ISCII91,x-ISO-2022-CN-CNS,x-ISO-2022-CN-GB,x-iso-8859-11,x-JIS0208,x-JISAutoDetect,x-Johab,x-MacArabic,x-MacCentralEurope,x-MacCroatian,x-MacCyrillic,x-MacDingbat,x-MacGreek,x-MacHebrew,x-MacIceland,x-MacRoman,x-MacRomania,x-MacSymbol,x-MacThai,x-MacTurkish,x-MacUkraine,x-MS932_0213,x-MS950-HKSCS,x-MS950-HKSCS-XP,x-mswin-936,x-PCK,x-SJIS_0213,x-UTF-16LE-BOM,X-UTF-32BE-BOM,X-UTF-32LE-BOM,x-windows-50220,x-windows-50221,x-windows-874,x-windows-949,x-windows-950,x-windows-iso2022jp ##### `task.count` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 1 *Validator:* [1,...] ##### `task.index` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 0 *Validator:* [0,...] #### Timestamps ##### `timestamp.mode` Determines how the connector will set the timestamp for the [ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field in `timestamp.field`. If set to `FILE_TIME` then the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used. *Importance:* MEDIUM *Type:* STRING *Default Value:* PROCESS_TIME *Validator:* Matches: ``FIELD``, ``FILE_TIME``, ``PROCESS_TIME`` ## [Json Source Connector](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir/sources/SpoolDirJsonSourceConnector.html) ``` com.github.jcustenborder.kafka.connect.spooldir.SpoolDirJsonSourceConnector ``` This connector is used to `stream ` JSON files from a directory while converting the data based on the schema supplied in the configuration. ### Important There are some caveats to running this connector with `schema.generation.enabled = true`. If schema generation is enabled the connector will start by reading one of the files that match `input.file.pattern` in the path specified by `input.path`. If there are no files when the connector starts or is restarted the connector will fail to start. If there are different fields in other files they will not be detected. The recommended path is to specify a schema that the files will be parsed with. This will ensure that data written by this connector to Kafka will be consistent across files that have inconsistent columns. For example if some files have an optional column that is not always included, create a schema that includes the column marked as optional. ### Note If you want to import JSON node by node in the file and do not care about schemas, do not use this connector with Schema Generation enabled. Take a look at the Schema Less Json Source Connector. ### Tip To get a starting point for a schema you can use the following command to generate an all String schema. This will give you the basic structure of a schema. From there you can changes the types to match what you expect. .. code-block:: bash mvn clean package export CLASSPATH="$(find target/kafka-connect-target/usr/share/kafka-connect/kafka-connect-spooldir -type f -name '*.jar' | tr '\n' ':')" kafka-run-class com.github.jcustenborder.kafka.connect.spooldir.AbstractSchemaGenerator -t json -f src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FieldsMatch.data -c config/JsonExample.properties -i id ### Configuration #### File System ##### `error.path` The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `input.file.pattern` Regular expression to check input file names against. This expression must match the entire filename. The equivalent of Matcher.matches(). *Importance:* HIGH *Type:* STRING ##### `input.path` The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `finished.path` The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING ##### `halt.on.error` Should the task halt when it encounters an error or continue to the next file. *Importance:* HIGH *Type:* BOOLEAN *Default Value:* true ##### `cleanup.policy` Determines how the connector should cleanup the files that have been successfully processed. NONE leaves the files in place which could cause them to be reprocessed if the connector is restarted. DELETE removes the file from the filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to a finished directory with subdirectories by date *Importance:* MEDIUM *Type:* STRING *Default Value:* MOVE *Validator:* Matches: ``NONE``, ``DELETE``, ``MOVE``, ``MOVEBYDATE`` ##### `task.partitioner` The task partitioner implementation is used when the connector is configured to use more than one task. This is used by each task to identify which files will be processed by that task. This ensures that each file is only assigned to one task. *Importance:* MEDIUM *Type:* STRING *Default Value:* ByName *Validator:* Matches: ``ByName`` ##### `file.buffer.size.bytes` The size of buffer for the BufferedInputStream that will be used to interact with the file system. *Importance:* LOW *Type:* INT *Default Value:* 131072 *Validator:* [1,...] ##### `file.minimum.age.ms` The amount of time in milliseconds after the file was last written to before the file can be processed. *Importance:* LOW *Type:* LONG *Default Value:* 0 *Validator:* [0,...] ##### `files.sort.attributes` The attributes each file will use to determine the sort order. `Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is the LastModified attribute of the file preferring older files first. *Importance:* LOW *Type:* LIST *Default Value:* [NameAsc] *Validator:* Matches: ``NameAsc``, ``NameDesc``, ``LengthAsc``, ``LengthDesc``, ``LastModifiedAsc``, ``LastModifiedDesc`` ##### `processing.file.extension` Before a file is processed, a flag is created in its directory to indicate the file is being handled. The flag file has the same name as the file, but with this property appended as a suffix. *Importance:* LOW *Type:* STRING *Default Value:* .PROCESSING *Validator:* Matches regex( ^.*\..+$ ) #### General ##### `topic` The Kafka topic to write the data to. *Importance:* HIGH *Type:* STRING ##### `batch.size` The number of records that should be returned with each batch. *Importance:* LOW *Type:* INT *Default Value:* 1000 ##### `empty.poll.wait.ms` The amount of time to wait if a poll returns an empty list of records. *Importance:* LOW *Type:* LONG *Default Value:* 500 *Validator:* [1,...,9223372036854775807] ##### `task.count` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 1 *Validator:* [1,...] ##### `task.index` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 0 *Validator:* [0,...] #### Schema ##### `key.schema` The schema for the key written to Kafka. *Importance:* HIGH *Type:* STRING ##### `value.schema` The schema for the value written to Kafka. *Importance:* HIGH *Type:* STRING #### Schema Generation ##### `schema.generation.enabled` Flag to determine if schemas should be dynamically generated. If set to true, `key.schema` and `value.schema` can be omitted, but `schema.generation.key.name` and `schema.generation.value.name` must be set. *Importance:* MEDIUM *Type:* BOOLEAN ##### `schema.generation.key.fields` The field(s) to use to build a key schema. This is only used during schema generation. *Importance:* MEDIUM *Type:* LIST ##### `schema.generation.key.name` The name of the generated key schema. *Importance:* MEDIUM *Type:* STRING *Default Value:* com.github.jcustenborder.kafka.connect.model.Key ##### `schema.generation.value.name` The name of the generated value schema. *Importance:* MEDIUM *Type:* STRING *Default Value:* com.github.jcustenborder.kafka.connect.model.Value #### Timestamps ##### `timestamp.field` The field in the value schema that will contain the parsed timestamp for the record. This field cannot be marked as optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html) *Importance:* MEDIUM *Type:* STRING ##### `timestamp.mode` Determines how the connector will set the timestamp for the [ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field in `timestamp.field`. If set to `FILE_TIME` then the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used. *Importance:* MEDIUM *Type:* STRING *Default Value:* PROCESS_TIME *Validator:* Matches: ``FIELD``, ``FILE_TIME``, ``PROCESS_TIME`` ##### `parser.timestamp.date.formats` The date formats that are expected in the file. This is a list of strings that will be used to parse the date fields in order. The most accurate date format should be the first in the list. Take a look at the Java documentation for more info. https://docs.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html *Importance:* LOW *Type:* LIST *Default Value:* [yyyy-MM-dd'T'HH:mm:ss, yyyy-MM-dd' 'HH:mm:ss] ##### `parser.timestamp.timezone` The timezone that all of the dates will be parsed with. *Importance:* LOW *Type:* STRING *Default Value:* UTC ## [Binary File Source Connector](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir/sources/SpoolDirBinaryFileSourceConnector.html) ``` com.github.jcustenborder.kafka.connect.spooldir.SpoolDirBinaryFileSourceConnector ``` This connector is used to read an entire file as a byte array write the data to Kafka. ### Warning Large files will be read as a single byte array. This means that the process could run out of memory or try to send a message to Kafka that is greater than the max message size. If this happens an exception will be thrown. ### Important The recommended converter to use is the ByteArrayConverter. Example: `value.converter=org.apache.kafka.connect.storage.ByteArrayConverter` ### Configuration #### File System ##### `error.path` The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `input.file.pattern` Regular expression to check input file names against. This expression must match the entire filename. The equivalent of Matcher.matches(). *Importance:* HIGH *Type:* STRING ##### `input.path` The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `finished.path` The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING ##### `halt.on.error` Should the task halt when it encounters an error or continue to the next file. *Importance:* HIGH *Type:* BOOLEAN *Default Value:* true ##### `cleanup.policy` Determines how the connector should cleanup the files that have been successfully processed. NONE leaves the files in place which could cause them to be reprocessed if the connector is restarted. DELETE removes the file from the filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to a finished directory with subdirectories by date *Importance:* MEDIUM *Type:* STRING *Default Value:* MOVE *Validator:* Matches: ``NONE``, ``DELETE``, ``MOVE``, ``MOVEBYDATE`` ##### `task.partitioner` The task partitioner implementation is used when the connector is configured to use more than one task. This is used by each task to identify which files will be processed by that task. This ensures that each file is only assigned to one task. *Importance:* MEDIUM *Type:* STRING *Default Value:* ByName *Validator:* Matches: ``ByName`` ##### `file.buffer.size.bytes` The size of buffer for the BufferedInputStream that will be used to interact with the file system. *Importance:* LOW *Type:* INT *Default Value:* 131072 *Validator:* [1,...] ##### `file.minimum.age.ms` The amount of time in milliseconds after the file was last written to before the file can be processed. *Importance:* LOW *Type:* LONG *Default Value:* 0 *Validator:* [0,...] ##### `files.sort.attributes` The attributes each file will use to determine the sort order. `Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is the LastModified attribute of the file preferring older files first. *Importance:* LOW *Type:* LIST *Default Value:* [NameAsc] *Validator:* Matches: ``NameAsc``, ``NameDesc``, ``LengthAsc``, ``LengthDesc``, ``LastModifiedAsc``, ``LastModifiedDesc`` ##### `processing.file.extension` Before a file is processed, a flag is created in its directory to indicate the file is being handled. The flag file has the same name as the file, but with this property appended as a suffix. *Importance:* LOW *Type:* STRING *Default Value:* .PROCESSING *Validator:* Matches regex( ^.*\..+$ ) #### General ##### `topic` The Kafka topic to write the data to. *Importance:* HIGH *Type:* STRING ##### `batch.size` The number of records that should be returned with each batch. *Importance:* LOW *Type:* INT *Default Value:* 1000 ##### `empty.poll.wait.ms` The amount of time to wait if a poll returns an empty list of records. *Importance:* LOW *Type:* LONG *Default Value:* 500 *Validator:* [1,...,9223372036854775807] ##### `task.count` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 1 *Validator:* [1,...] ##### `task.index` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 0 *Validator:* [0,...] #### Timestamps ##### `timestamp.mode` Determines how the connector will set the timestamp for the [ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field in `timestamp.field`. If set to `FILE_TIME` then the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used. *Importance:* MEDIUM *Type:* STRING *Default Value:* PROCESS_TIME *Validator:* Matches: ``FIELD``, ``FILE_TIME``, ``PROCESS_TIME`` ## [Line Delimited Source Connector](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir/sources/SpoolDirLineDelimitedSourceConnector.html) ``` com.github.jcustenborder.kafka.connect.spooldir.SpoolDirLineDelimitedSourceConnector ``` This connector is used to read a file line by line and write the data to Kafka. ### Important The recommended converter to use is the StringConverter. Example: `value.converter=org.apache.kafka.connect.storage.StringConverter` ### Configuration #### File System ##### `error.path` The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `input.file.pattern` Regular expression to check input file names against. This expression must match the entire filename. The equivalent of Matcher.matches(). *Importance:* HIGH *Type:* STRING ##### `input.path` The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `finished.path` The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING ##### `halt.on.error` Should the task halt when it encounters an error or continue to the next file. *Importance:* HIGH *Type:* BOOLEAN *Default Value:* true ##### `cleanup.policy` Determines how the connector should cleanup the files that have been successfully processed. NONE leaves the files in place which could cause them to be reprocessed if the connector is restarted. DELETE removes the file from the filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to a finished directory with subdirectories by date *Importance:* MEDIUM *Type:* STRING *Default Value:* MOVE *Validator:* Matches: ``NONE``, ``DELETE``, ``MOVE``, ``MOVEBYDATE`` ##### `task.partitioner` The task partitioner implementation is used when the connector is configured to use more than one task. This is used by each task to identify which files will be processed by that task. This ensures that each file is only assigned to one task. *Importance:* MEDIUM *Type:* STRING *Default Value:* ByName *Validator:* Matches: ``ByName`` ##### `file.buffer.size.bytes` The size of buffer for the BufferedInputStream that will be used to interact with the file system. *Importance:* LOW *Type:* INT *Default Value:* 131072 *Validator:* [1,...] ##### `file.minimum.age.ms` The amount of time in milliseconds after the file was last written to before the file can be processed. *Importance:* LOW *Type:* LONG *Default Value:* 0 *Validator:* [0,...] ##### `files.sort.attributes` The attributes each file will use to determine the sort order. `Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is the LastModified attribute of the file preferring older files first. *Importance:* LOW *Type:* LIST *Default Value:* [NameAsc] *Validator:* Matches: ``NameAsc``, ``NameDesc``, ``LengthAsc``, ``LengthDesc``, ``LastModifiedAsc``, ``LastModifiedDesc`` ##### `processing.file.extension` Before a file is processed, a flag is created in its directory to indicate the file is being handled. The flag file has the same name as the file, but with this property appended as a suffix. *Importance:* LOW *Type:* STRING *Default Value:* .PROCESSING *Validator:* Matches regex( ^.*\..+$ ) #### General ##### `topic` The Kafka topic to write the data to. *Importance:* HIGH *Type:* STRING ##### `batch.size` The number of records that should be returned with each batch. *Importance:* LOW *Type:* INT *Default Value:* 1000 ##### `empty.poll.wait.ms` The amount of time to wait if a poll returns an empty list of records. *Importance:* LOW *Type:* LONG *Default Value:* 500 *Validator:* [1,...,9223372036854775807] ##### `file.charset` Character set to read wth file with. *Importance:* LOW *Type:* STRING *Default Value:* UTF-8 *Validator:* Big5,Big5-HKSCS,CESU-8,EUC-JP,EUC-KR,GB18030,GB2312,GBK,IBM-Thai,IBM00858,IBM01140,IBM01141,IBM01142,IBM01143,IBM01144,IBM01145,IBM01146,IBM01147,IBM01148,IBM01149,IBM037,IBM1026,IBM1047,IBM273,IBM277,IBM278,IBM280,IBM284,IBM285,IBM290,IBM297,IBM420,IBM424,IBM437,IBM500,IBM775,IBM850,IBM852,IBM855,IBM857,IBM860,IBM861,IBM862,IBM863,IBM864,IBM865,IBM866,IBM868,IBM869,IBM870,IBM871,IBM918,ISO-2022-CN,ISO-2022-JP,ISO-2022-JP-2,ISO-2022-KR,ISO-8859-1,ISO-8859-13,ISO-8859-15,ISO-8859-16,ISO-8859-2,ISO-8859-3,ISO-8859-4,ISO-8859-5,ISO-8859-6,ISO-8859-7,ISO-8859-8,ISO-8859-9,JIS_X0201,JIS_X0212-1990,KOI8-R,KOI8-U,Shift_JIS,TIS-620,US-ASCII,UTF-16,UTF-16BE,UTF-16LE,UTF-32,UTF-32BE,UTF-32LE,UTF-8,windows-1250,windows-1251,windows-1252,windows-1253,windows-1254,windows-1255,windows-1256,windows-1257,windows-1258,windows-31j,x-Big5-HKSCS-2001,x-Big5-Solaris,x-euc-jp-linux,x-EUC-TW,x-eucJP-Open,x-IBM1006,x-IBM1025,x-IBM1046,x-IBM1097,x-IBM1098,x-IBM1112,x-IBM1122,x-IBM1123,x-IBM1124,x-IBM1129,x-IBM1166,x-IBM1364,x-IBM1381,x-IBM1383,x-IBM29626C,x-IBM300,x-IBM33722,x-IBM737,x-IBM833,x-IBM834,x-IBM856,x-IBM874,x-IBM875,x-IBM921,x-IBM922,x-IBM930,x-IBM933,x-IBM935,x-IBM937,x-IBM939,x-IBM942,x-IBM942C,x-IBM943,x-IBM943C,x-IBM948,x-IBM949,x-IBM949C,x-IBM950,x-IBM964,x-IBM970,x-ISCII91,x-ISO-2022-CN-CNS,x-ISO-2022-CN-GB,x-iso-8859-11,x-JIS0208,x-JISAutoDetect,x-Johab,x-MacArabic,x-MacCentralEurope,x-MacCroatian,x-MacCyrillic,x-MacDingbat,x-MacGreek,x-MacHebrew,x-MacIceland,x-MacRoman,x-MacRomania,x-MacSymbol,x-MacThai,x-MacTurkish,x-MacUkraine,x-MS932_0213,x-MS950-HKSCS,x-MS950-HKSCS-XP,x-mswin-936,x-PCK,x-SJIS_0213,x-UTF-16LE-BOM,X-UTF-32BE-BOM,X-UTF-32LE-BOM,x-windows-50220,x-windows-50221,x-windows-874,x-windows-949,x-windows-950,x-windows-iso2022jp ##### `task.count` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 1 *Validator:* [1,...] ##### `task.index` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 0 *Validator:* [0,...] #### Timestamps ##### `timestamp.mode` Determines how the connector will set the timestamp for the [ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field in `timestamp.field`. If set to `FILE_TIME` then the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used. *Importance:* MEDIUM *Type:* STRING *Default Value:* PROCESS_TIME *Validator:* Matches: ``FIELD``, ``FILE_TIME``, ``PROCESS_TIME`` ## [Avro Source Connector](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir/sources/SpoolDirAvroSourceConnector.html) ``` com.github.jcustenborder.kafka.connect.spooldir.SpoolDirAvroSourceConnector ``` This connector is used to read avro data files from the file system and write their contents to kafka. ### Configuration #### File System ##### `error.path` The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `input.file.pattern` Regular expression to check input file names against. This expression must match the entire filename. The equivalent of Matcher.matches(). *Importance:* HIGH *Type:* STRING ##### `input.path` The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `finished.path` The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING ##### `halt.on.error` Should the task halt when it encounters an error or continue to the next file. *Importance:* HIGH *Type:* BOOLEAN *Default Value:* true ##### `cleanup.policy` Determines how the connector should cleanup the files that have been successfully processed. NONE leaves the files in place which could cause them to be reprocessed if the connector is restarted. DELETE removes the file from the filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to a finished directory with subdirectories by date *Importance:* MEDIUM *Type:* STRING *Default Value:* MOVE *Validator:* Matches: ``NONE``, ``DELETE``, ``MOVE``, ``MOVEBYDATE`` ##### `task.partitioner` The task partitioner implementation is used when the connector is configured to use more than one task. This is used by each task to identify which files will be processed by that task. This ensures that each file is only assigned to one task. *Importance:* MEDIUM *Type:* STRING *Default Value:* ByName *Validator:* Matches: ``ByName`` ##### `file.buffer.size.bytes` The size of buffer for the BufferedInputStream that will be used to interact with the file system. *Importance:* LOW *Type:* INT *Default Value:* 131072 *Validator:* [1,...] ##### `file.minimum.age.ms` The amount of time in milliseconds after the file was last written to before the file can be processed. *Importance:* LOW *Type:* LONG *Default Value:* 0 *Validator:* [0,...] ##### `files.sort.attributes` The attributes each file will use to determine the sort order. `Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is the LastModified attribute of the file preferring older files first. *Importance:* LOW *Type:* LIST *Default Value:* [NameAsc] *Validator:* Matches: ``NameAsc``, ``NameDesc``, ``LengthAsc``, ``LengthDesc``, ``LastModifiedAsc``, ``LastModifiedDesc`` ##### `processing.file.extension` Before a file is processed, a flag is created in its directory to indicate the file is being handled. The flag file has the same name as the file, but with this property appended as a suffix. *Importance:* LOW *Type:* STRING *Default Value:* .PROCESSING *Validator:* Matches regex( ^.*\..+$ ) #### General ##### `topic` The Kafka topic to write the data to. *Importance:* HIGH *Type:* STRING ##### `batch.size` The number of records that should be returned with each batch. *Importance:* LOW *Type:* INT *Default Value:* 1000 ##### `empty.poll.wait.ms` The amount of time to wait if a poll returns an empty list of records. *Importance:* LOW *Type:* LONG *Default Value:* 500 *Validator:* [1,...,9223372036854775807] ##### `task.count` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 1 *Validator:* [1,...] ##### `task.index` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 0 *Validator:* [0,...] #### Timestamps ##### `timestamp.mode` Determines how the connector will set the timestamp for the [ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field in `timestamp.field`. If set to `FILE_TIME` then the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used. *Importance:* MEDIUM *Type:* STRING *Default Value:* PROCESS_TIME *Validator:* Matches: ``FIELD``, ``FILE_TIME``, ``PROCESS_TIME`` ## [Extended Log File Format Source Connector](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir/sources/SpoolDirELFSourceConnector.html) ``` com.github.jcustenborder.kafka.connect.spooldir.elf.SpoolDirELFSourceConnector ``` This connector is used to stream `Extended Log File Format ` files from a directory while converting the data to a strongly typed schema. ### Configuration #### File System ##### `error.path` The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `input.file.pattern` Regular expression to check input file names against. This expression must match the entire filename. The equivalent of Matcher.matches(). *Importance:* HIGH *Type:* STRING ##### `input.path` The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `finished.path` The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING ##### `halt.on.error` Should the task halt when it encounters an error or continue to the next file. *Importance:* HIGH *Type:* BOOLEAN *Default Value:* true ##### `cleanup.policy` Determines how the connector should cleanup the files that have been successfully processed. NONE leaves the files in place which could cause them to be reprocessed if the connector is restarted. DELETE removes the file from the filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to a finished directory with subdirectories by date *Importance:* MEDIUM *Type:* STRING *Default Value:* MOVE *Validator:* Matches: ``NONE``, ``DELETE``, ``MOVE``, ``MOVEBYDATE`` ##### `task.partitioner` The task partitioner implementation is used when the connector is configured to use more than one task. This is used by each task to identify which files will be processed by that task. This ensures that each file is only assigned to one task. *Importance:* MEDIUM *Type:* STRING *Default Value:* ByName *Validator:* Matches: ``ByName`` ##### `file.buffer.size.bytes` The size of buffer for the BufferedInputStream that will be used to interact with the file system. *Importance:* LOW *Type:* INT *Default Value:* 131072 *Validator:* [1,...] ##### `file.minimum.age.ms` The amount of time in milliseconds after the file was last written to before the file can be processed. *Importance:* LOW *Type:* LONG *Default Value:* 0 *Validator:* [0,...] ##### `files.sort.attributes` The attributes each file will use to determine the sort order. `Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is the LastModified attribute of the file preferring older files first. *Importance:* LOW *Type:* LIST *Default Value:* [NameAsc] *Validator:* Matches: ``NameAsc``, ``NameDesc``, ``LengthAsc``, ``LengthDesc``, ``LastModifiedAsc``, ``LastModifiedDesc`` ##### `processing.file.extension` Before a file is processed, a flag is created in its directory to indicate the file is being handled. The flag file has the same name as the file, but with this property appended as a suffix. *Importance:* LOW *Type:* STRING *Default Value:* .PROCESSING *Validator:* Matches regex( ^.*\..+$ ) #### General ##### `topic` The Kafka topic to write the data to. *Importance:* HIGH *Type:* STRING ##### `batch.size` The number of records that should be returned with each batch. *Importance:* LOW *Type:* INT *Default Value:* 1000 ##### `empty.poll.wait.ms` The amount of time to wait if a poll returns an empty list of records. *Importance:* LOW *Type:* LONG *Default Value:* 500 *Validator:* [1,...,9223372036854775807] ##### `task.count` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 1 *Validator:* [1,...] ##### `task.index` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 0 *Validator:* [0,...] #### Timestamps ##### `timestamp.mode` Determines how the connector will set the timestamp for the [ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field in `timestamp.field`. If set to `FILE_TIME` then the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used. *Importance:* MEDIUM *Type:* STRING *Default Value:* PROCESS_TIME *Validator:* Matches: ``FIELD``, ``FILE_TIME``, ``PROCESS_TIME`` ## [CSV Source Connector](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-spooldir/sources/SpoolDirCsvSourceConnector.html) ``` com.github.jcustenborder.kafka.connect.spooldir.SpoolDirCsvSourceConnector ``` The SpoolDirCsvSourceConnector will monitor the directory specified in `input.path` for files and read them as a CSV converting each of the records to the strongly typed equivalent specified in `key.schema` and `value.schema`. ### Important There are some caveats to running this connector with `schema.generation.enabled = true`. If schema generation is enabled the connector will start by reading one of the files that match `input.file.pattern` in the path specified by `input.path`. If there are no files when the connector starts or is restarted the connector will fail to start. If there are different fields in other files they will not be detected. The recommended path is to specify a schema that the files will be parsed with. This will ensure that data written by this connector to Kafka will be consistent across files that have inconsistent columns. For example if some files have an optional column that is not always included, create a schema that includes the column marked as optional. ### Tip To get a starting point for a schema you can use the following command to generate an all String schema. This will give you the basic structure of a schema. From there you can changes the types to match what you expect. .. code-block:: bash mvn clean package export CLASSPATH="$(find target/kafka-connect-target/usr/share/kafka-connect/kafka-connect-spooldir -type f -name '*.jar' | tr '\n' ':')" kafka-run-class com.github.jcustenborder.kafka.connect.spooldir.AbstractSchemaGenerator -t csv -f src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.data -c config/CSVExample.properties -i id ### Configuration #### File System ##### `error.path` The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `input.file.pattern` Regular expression to check input file names against. This expression must match the entire filename. The equivalent of Matcher.matches(). *Importance:* HIGH *Type:* STRING ##### `input.path` The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING *Validator:* Absolute path to a directory that exists and is writable. ##### `finished.path` The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect. *Importance:* HIGH *Type:* STRING ##### `halt.on.error` Should the task halt when it encounters an error or continue to the next file. *Importance:* HIGH *Type:* BOOLEAN *Default Value:* true ##### `cleanup.policy` Determines how the connector should cleanup the files that have been successfully processed. NONE leaves the files in place which could cause them to be reprocessed if the connector is restarted. DELETE removes the file from the filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to a finished directory with subdirectories by date *Importance:* MEDIUM *Type:* STRING *Default Value:* MOVE *Validator:* Matches: ``NONE``, ``DELETE``, ``MOVE``, ``MOVEBYDATE`` ##### `task.partitioner` The task partitioner implementation is used when the connector is configured to use more than one task. This is used by each task to identify which files will be processed by that task. This ensures that each file is only assigned to one task. *Importance:* MEDIUM *Type:* STRING *Default Value:* ByName *Validator:* Matches: ``ByName`` ##### `file.buffer.size.bytes` The size of buffer for the BufferedInputStream that will be used to interact with the file system. *Importance:* LOW *Type:* INT *Default Value:* 131072 *Validator:* [1,...] ##### `file.minimum.age.ms` The amount of time in milliseconds after the file was last written to before the file can be processed. *Importance:* LOW *Type:* LONG *Default Value:* 0 *Validator:* [0,...] ##### `files.sort.attributes` The attributes each file will use to determine the sort order. `Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is the LastModified attribute of the file preferring older files first. *Importance:* LOW *Type:* LIST *Default Value:* [NameAsc] *Validator:* Matches: ``NameAsc``, ``NameDesc``, ``LengthAsc``, ``LengthDesc``, ``LastModifiedAsc``, ``LastModifiedDesc`` ##### `processing.file.extension` Before a file is processed, a flag is created in its directory to indicate the file is being handled. The flag file has the same name as the file, but with this property appended as a suffix. *Importance:* LOW *Type:* STRING *Default Value:* .PROCESSING *Validator:* Matches regex( ^.*\..+$ ) #### General ##### `topic` The Kafka topic to write the data to. *Importance:* HIGH *Type:* STRING ##### `batch.size` The number of records that should be returned with each batch. *Importance:* LOW *Type:* INT *Default Value:* 1000 ##### `csv.case.sensitive.field.names` Flag to determine if the field names in the header row should be treated as case sensitive. *Importance:* LOW *Type:* BOOLEAN ##### `csv.rfc.4180.parser.enabled` Flag to determine if the RFC 4180 parser should be used instead of the default parser. *Importance:* LOW *Type:* BOOLEAN ##### `empty.poll.wait.ms` The amount of time to wait if a poll returns an empty list of records. *Importance:* LOW *Type:* LONG *Default Value:* 500 *Validator:* [1,...,9223372036854775807] ##### `task.count` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 1 *Validator:* [1,...] ##### `task.index` Internal setting to the connector used to instruct a task on which files to select. The connector will override this setting. *Importance:* LOW *Type:* INT *Default Value:* 0 *Validator:* [0,...] #### Schema ##### `key.schema` The schema for the key written to Kafka. *Importance:* HIGH *Type:* STRING ##### `value.schema` The schema for the value written to Kafka. *Importance:* HIGH *Type:* STRING #### CSV Parsing ##### `csv.first.row.as.header` Flag to indicate if the fist row of data contains the header of the file. If true the position of the columns will be determined by the first row to the CSV. The column position will be inferred from the position of the schema supplied in `value.schema`. If set to true the number of columns must be greater than or equal to the number of fields in the schema. *Importance:* MEDIUM *Type:* BOOLEAN ##### `csv.escape.char` The character as an integer to use when a special character is encountered. The default escape character is typically a \(92) *Importance:* LOW *Type:* INT *Default Value:* 92 ##### `csv.file.charset` Character set to read wth file with. *Importance:* LOW *Type:* STRING *Default Value:* UTF-8 *Validator:* Big5,Big5-HKSCS,CESU-8,EUC-JP,EUC-KR,GB18030,GB2312,GBK,IBM-Thai,IBM00858,IBM01140,IBM01141,IBM01142,IBM01143,IBM01144,IBM01145,IBM01146,IBM01147,IBM01148,IBM01149,IBM037,IBM1026,IBM1047,IBM273,IBM277,IBM278,IBM280,IBM284,IBM285,IBM290,IBM297,IBM420,IBM424,IBM437,IBM500,IBM775,IBM850,IBM852,IBM855,IBM857,IBM860,IBM861,IBM862,IBM863,IBM864,IBM865,IBM866,IBM868,IBM869,IBM870,IBM871,IBM918,ISO-2022-CN,ISO-2022-JP,ISO-2022-JP-2,ISO-2022-KR,ISO-8859-1,ISO-8859-13,ISO-8859-15,ISO-8859-16,ISO-8859-2,ISO-8859-3,ISO-8859-4,ISO-8859-5,ISO-8859-6,ISO-8859-7,ISO-8859-8,ISO-8859-9,JIS_X0201,JIS_X0212-1990,KOI8-R,KOI8-U,Shift_JIS,TIS-620,US-ASCII,UTF-16,UTF-16BE,UTF-16LE,UTF-32,UTF-32BE,UTF-32LE,UTF-8,windows-1250,windows-1251,windows-1252,windows-1253,windows-1254,windows-1255,windows-1256,windows-1257,windows-1258,windows-31j,x-Big5-HKSCS-2001,x-Big5-Solaris,x-euc-jp-linux,x-EUC-TW,x-eucJP-Open,x-IBM1006,x-IBM1025,x-IBM1046,x-IBM1097,x-IBM1098,x-IBM1112,x-IBM1122,x-IBM1123,x-IBM1124,x-IBM1129,x-IBM1166,x-IBM1364,x-IBM1381,x-IBM1383,x-IBM29626C,x-IBM300,x-IBM33722,x-IBM737,x-IBM833,x-IBM834,x-IBM856,x-IBM874,x-IBM875,x-IBM921,x-IBM922,x-IBM930,x-IBM933,x-IBM935,x-IBM937,x-IBM939,x-IBM942,x-IBM942C,x-IBM943,x-IBM943C,x-IBM948,x-IBM949,x-IBM949C,x-IBM950,x-IBM964,x-IBM970,x-ISCII91,x-ISO-2022-CN-CNS,x-ISO-2022-CN-GB,x-iso-8859-11,x-JIS0208,x-JISAutoDetect,x-Johab,x-MacArabic,x-MacCentralEurope,x-MacCroatian,x-MacCyrillic,x-MacDingbat,x-MacGreek,x-MacHebrew,x-MacIceland,x-MacRoman,x-MacRomania,x-MacSymbol,x-MacThai,x-MacTurkish,x-MacUkraine,x-MS932_0213,x-MS950-HKSCS,x-MS950-HKSCS-XP,x-mswin-936,x-PCK,x-SJIS_0213,x-UTF-16LE-BOM,X-UTF-32BE-BOM,X-UTF-32LE-BOM,x-windows-50220,x-windows-50221,x-windows-874,x-windows-949,x-windows-950,x-windows-iso2022jp ##### `csv.ignore.leading.whitespace` Sets the ignore leading whitespace setting - if true, white space in front of a quote in a field is ignored. *Importance:* LOW *Type:* BOOLEAN *Default Value:* true ##### `csv.ignore.quotations` Sets the ignore quotations mode - if true, quotations are ignored. *Importance:* LOW *Type:* BOOLEAN ##### `csv.keep.carriage.return` Flag to determine if the carriage return at the end of the line should be maintained. *Importance:* LOW *Type:* BOOLEAN ##### `csv.null.field.indicator` Indicator to determine how the CSV Reader can determine if a field is null. Valid values are EMPTY_SEPARATORS, EMPTY_QUOTES, BOTH, NEITHER. For more information see http://opencsv.sourceforge.net/apidocs/com/opencsv/enums/CSVReaderNullFieldIndicator.html. *Importance:* LOW *Type:* STRING *Default Value:* NEITHER *Validator:* Matches: ``EMPTY_SEPARATORS``, ``EMPTY_QUOTES``, ``BOTH``, ``NEITHER`` ##### `csv.quote.char` The character that is used to quote a field. This typically happens when the csv.separator.char character is within the data. *Importance:* LOW *Type:* INT *Default Value:* 34 ##### `csv.separator.char` The character that separates each field in the form of an integer. Typically in a CSV this is a ,(44) character. A TSV would use a tab(9) character. If `csv.separator.char` is defined as a null(0), then the RFC 4180 parser must be utilized by default. This is the equivalent of `csv.rfc.4180.parser.enabled = true`. *Importance:* LOW *Type:* INT *Default Value:* 44 ##### `csv.skip.lines` Number of lines to skip in the beginning of the file. *Importance:* LOW *Type:* INT *Default Value:* 0 ##### `csv.strict.quotes` Sets the strict quotes setting - if true, characters outside the quotes are ignored. *Importance:* LOW *Type:* BOOLEAN ##### `csv.verify.reader` Flag to determine if the reader should be verified. *Importance:* LOW *Type:* BOOLEAN *Default Value:* true #### Schema Generation ##### `schema.generation.enabled` Flag to determine if schemas should be dynamically generated. If set to true, `key.schema` and `value.schema` can be omitted, but `schema.generation.key.name` and `schema.generation.value.name` must be set. *Importance:* MEDIUM *Type:* BOOLEAN ##### `schema.generation.key.fields` The field(s) to use to build a key schema. This is only used during schema generation. *Importance:* MEDIUM *Type:* LIST ##### `schema.generation.key.name` The name of the generated key schema. *Importance:* MEDIUM *Type:* STRING *Default Value:* com.github.jcustenborder.kafka.connect.model.Key ##### `schema.generation.value.name` The name of the generated value schema. *Importance:* MEDIUM *Type:* STRING *Default Value:* com.github.jcustenborder.kafka.connect.model.Value #### Timestamps ##### `timestamp.field` The field in the value schema that will contain the parsed timestamp for the record. This field cannot be marked as optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html) *Importance:* MEDIUM *Type:* STRING ##### `timestamp.mode` Determines how the connector will set the timestamp for the [ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field in `timestamp.field`. If set to `FILE_TIME` then the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used. *Importance:* MEDIUM *Type:* STRING *Default Value:* PROCESS_TIME *Validator:* Matches: ``FIELD``, ``FILE_TIME``, ``PROCESS_TIME`` ##### `parser.timestamp.date.formats` The date formats that are expected in the file. This is a list of strings that will be used to parse the date fields in order. The most accurate date format should be the first in the list. Take a look at the Java documentation for more info. https://docs.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html *Importance:* LOW *Type:* LIST *Default Value:* [yyyy-MM-dd'T'HH:mm:ss, yyyy-MM-dd' 'HH:mm:ss] ##### `parser.timestamp.timezone` The timezone that all of the dates will be parsed with. *Importance:* LOW *Type:* STRING *Default Value:* UTC # Development ## Building the source ```bash mvn clean package ``` ## Contributions Contributions are always welcomed! Before you start any development please create an issue and start a discussion. Create a pull request against your newly created issue and we're happy to see if we can merge your pull request. First and foremost any time you're adding code to the code base you need to include test coverage. Make sure that you run `mvn clean package` before submitting your pull to ensure that all of the tests, checkstyle rules, and the package can be successfully built. ================================================ FILE: bin/debug.sh ================================================ #!/usr/bin/env bash # # Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # : ${INPUT_PATH:='/tmp/spooldir/input'} : ${ERROR_PATH:='/tmp/spooldir/error'} : ${FINISHED_PATH:='/tmp/spooldir/finished'} : ${DEBUG_SUSPEND_FLAG:='y'} export KAFKA_DEBUG='n' export DEBUG_SUSPEND_FLAG='n' # export KAFKA_OPTS='-agentpath:/Applications/YourKit-Java-Profiler-2017.02.app/Contents/Resources/bin/mac/libyjpagent.jnilib=disablestacktelemetry,exceptions=disable,delay=10000' set -e # mvn clean package if [ ! -d "${INPUT_PATH}" ]; then mkdir -p "${INPUT_PATH}" fi if [ ! -d "${ERROR_PATH}" ]; then mkdir -p "${ERROR_PATH}" fi if [ ! -d "${FINISHED_PATH}" ]; then mkdir -p "${FINISHED_PATH}" fi cp /Users/jeremy/Downloads/csv-spooldir-source.csv "${INPUT_PATH}/csv-spooldir-source.csv" # cp src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.data "${INPUT_PATH}/FieldsMatch.csv" # cp src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FieldsMatch.data "${INPUT_PATH}/FieldsMatch.json" # connect-standalone config/connect-avro-docker.properties config/CSVSchemaGenerator.properties # connect-standalone config/connect-avro-docker.properties config/JsonExample.properties # connect-standalone config/connect-avro-docker.properties config/AvroExample.properties export DOCKER_IMAGE="confluentinc/cp-kafka-connect:5.5.2-1-ubi8" docker run --rm --network=kafka-connect-spooldir_default \ -p "5005:5005" \ -v "/tmp/spooldir:/tmp/spooldir" \ -v "$(pwd)/config:/config" \ -v "$(pwd)/target/kafka-connect-target/usr/share/kafka-connect:/plugins" \ "${DOCKER_IMAGE}" /bin/connect-standalone /config/connect-avro-docker.properties /config/CSVSchemaGenerator.properties ================================================ FILE: config/AvroExample.properties ================================================ # # Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # name=AvroSpoolDir tasks.max=1 connector.class=com.github.jcustenborder.kafka.connect.spooldir.SpoolDirAvroSourceConnector input.file.pattern=^.*\.avro$ halt.on.error=false topic=testing input.path=/Users/jeremy/data/stackoverflow finished.path=/tmp/spooldir/finished error.path=/tmp/spooldir/error batch.size = 5000 cleanup.policy = NONE ================================================ FILE: config/CSVExample.json ================================================ { "name": "CsvSpoolDir", "config": { "tasks.max": "1", "connector.class": "com.github.jcustenborder.kafka.connect.spooldir.SpoolDirCsvSourceConnector", "input.file.pattern": "^.*\\.csv$", "halt.on.error": "false", "topic": "testing" "csv.first.row.as.header": "true", "csv.null.field.indicator": "EMPTY_SEPARATORS", "input.path": "/tmp/spooldir/input", "finished.path": "/tmp/spooldir/finished", "error.path": "/tmp/spooldir/error", "key.schema": "{\"name\":\"com.example.users.UserKey\",\"type\":\"STRUCT\",\"isOptional\":false,\"fieldSchemas\":{\"id\":{\"type\":\"INT64\",\"isOptional\":false}}}", "value.schema": "{\"name\":\"com.example.users.User\",\"type\":\"STRUCT\",\"isOptional\":false,\"fieldSchemas\":{\"id\":{\"type\":\"INT64\",\"isOptional\":false},\"first_name\":{\"type\":\"STRING\",\"isOptional\":true},\"last_name\":{\"type\":\"STRING\",\"isOptional\":true},\"email\":{\"type\":\"STRING\",\"isOptional\":true},\"gender\":{\"type\":\"STRING\",\"isOptional\":true},\"ip_address\":{\"type\":\"STRING\",\"isOptional\":true},\"last_login\":{\"name\":\"org.apache.kafka.connect.data.Timestamp\",\"type\":\"INT64\",\"version\":1,\"isOptional\":true},\"account_balance\":{\"name\":\"org.apache.kafka.connect.data.Decimal\",\"type\":\"BYTES\",\"version\":1,\"parameters\":{\"scale\":\"2\"},\"isOptional\":true},\"country\":{\"type\":\"STRING\",\"isOptional\":true},\"favorite_color\":{\"type\":\"STRING\",\"isOptional\":true}}}" } } ================================================ FILE: config/CSVExample.properties ================================================ # # Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # name=CsvSpoolDir tasks.max=1 connector.class=com.github.jcustenborder.kafka.connect.spooldir.SpoolDirCsvSourceConnector input.file.pattern=^.*\.csv$ halt.on.error=false topic=testing key.schema={"name":"com.example.users.UserKey","type":"STRUCT","isOptional":false,"fieldSchemas":{"id":{"type":"INT64","isOptional":false}}} value.schema={"name":"com.example.users.User","type":"STRUCT","isOptional":false,"fieldSchemas":{"id":{"type":"INT64","isOptional":false},"first_name":{"type":"STRING","isOptional":true},"last_name":{"type":"STRING","isOptional":true},"email":{"type":"STRING","isOptional":true},"gender":{"type":"STRING","isOptional":true},"ip_address":{"type":"STRING","isOptional":true},"last_login":{"name":"org.apache.kafka.connect.data.Timestamp","type":"INT64","version":1,"isOptional":true},"account_balance":{"name":"org.apache.kafka.connect.data.Decimal","type":"BYTES","version":1,"parameters":{"scale":"2"},"isOptional":true},"country":{"type":"STRING","isOptional":true},"favorite_color":{"type":"STRING","isOptional":true}}} csv.first.row.as.header=true csv.null.field.indicator=EMPTY_SEPARATORS input.path=/tmp/spooldir/input finished.path=/tmp/spooldir/finished error.path=/tmp/spooldir/error batch.size = 5000 cleanup.policy = DELETE file.buffer.size.bytes = 1048576 ================================================ FILE: config/CSVSchemaGenerator.properties ================================================ # # Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # name=CsvSpoolDir tasks.max=1 connector.class=com.github.jcustenborder.kafka.connect.spooldir.SpoolDirCsvSourceConnector input.file.pattern=^.*\.csv$ halt.on.error=false topic=testing csv.first.row.as.header=true csv.null.field.indicator=EMPTY_SEPARATORS input.path=/tmp/spooldir/input finished.path=/tmp/spooldir/finished error.path=/tmp/spooldir/error batch.size = 5000 cleanup.policy = DELETE file.buffer.size.bytes = 1048576 schema.generation.enabled=true ================================================ FILE: config/JsonExample.properties ================================================ # # Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # name=JsonSpoolDir tasks.max=1 connector.class=com.github.jcustenborder.kafka.connect.spooldir.SpoolDirJsonSourceConnector input.file.pattern=^.*\.json$ finished.path=/tmp/spooldir/finished input.path=/tmp/spooldir/input error.path=/tmp/spooldir/error halt.on.error=false topic=testing key.schema={"name":"com.example.users.UserKey","type":"STRUCT","isOptional":false,"fieldSchemas":{"id":{"type":"INT64","isOptional":false}}} value.schema={"name":"com.example.users.User","type":"STRUCT","isOptional":false,"fieldSchemas":{"id":{"type":"INT64","isOptional":false},"first_name":{"type":"STRING","isOptional":true},"last_name":{"type":"STRING","isOptional":true},"email":{"type":"STRING","isOptional":true},"gender":{"type":"STRING","isOptional":true},"ip_address":{"type":"STRING","isOptional":true},"last_login":{"name":"org.apache.kafka.connect.data.Timestamp","type":"INT64","version":1,"isOptional":true},"account_balance":{"name":"org.apache.kafka.connect.data.Decimal","type":"BYTES","version":1,"parameters":{"scale":"2"},"isOptional":true},"country":{"type":"STRING","isOptional":true},"favorite_color":{"type":"STRING","isOptional":true}}} ================================================ FILE: config/connect-avro-docker.properties ================================================ # # Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # group.id=foo bootstrap.servers=kafka:9092 key.converter=io.confluent.connect.avro.AvroConverter key.converter.schema.registry.url=http://schema-registry:8081 value.converter=io.confluent.connect.avro.AvroConverter value.converter.schema.registry.url=http://schema-registry:8081 internal.key.converter=org.apache.kafka.connect.json.JsonConverter internal.value.converter=org.apache.kafka.connect.json.JsonConverter internal.key.converter.schemas.enable=false internal.value.converter.schemas.enable=false offset.storage.file.filename=/tmp/connect.offsets plugin.path=target/kafka-connect-target/usr/share/kafka-connect,/plugins config.storage.replication.factor=1 config.storage.topic=connect_config offset.storage.replication.factor=1 offset.storage.topic=connect_offset status.storage.replication.factor=1 status.storage.topic=connect_status ================================================ FILE: docker-compose.yml ================================================ # # Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # version: "2" services: zookeeper: image: confluentinc/cp-zookeeper:6.0.0 ports: - "2181:2181" environment: ZOOKEEPER_CLIENT_PORT: 2181 kafka: image: confluentinc/cp-kafka:6.0.0 depends_on: - zookeeper ports: - "9092:9092" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" KAFKA_ADVERTISED_LISTENERS: "plaintext://kafka:9092" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 schema-registry: image: confluentinc/cp-schema-registry:6.0.0 depends_on: - kafka - zookeeper ports: - "8081:8081" environment: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" SCHEMA_REGISTRY_HOST_NAME: schema-registry connect: image: confluentinc/cp-kafka-connect:5.5.1-1-ubi8 depends_on: - kafka - zookeeper - schema-registry ports: - "5005:5005" environment: CONNECT_BOOTSTRAP_SERVERS: kafka:9092 CONNECT_GROUP_ID: spooldir ================================================ FILE: pom.xml ================================================ 4.0.0 com.github.jcustenborder.kafka.connect kafka-connect-parent 2.8.0-1 kafka-connect-spooldir 2.0-SNAPSHOT kafka-connect-spooldir A Kafka Connect connector reading delimited files from the file system. https://github.com/jcustenborder/kafka-connect-spooldir 2016 Apache License 2.0 https:/github.com/jcustenborder/kafka-connect-spooldir/LICENSE repo jcustenborder Jeremy Custenborder https://github.com/jcustenborder Committer scm:git:https://github.com/jcustenborder/kafka-connect-spooldir.git scm:git:git@github.com:jcustenborder/kafka-connect-spooldir.git https://github.com/jcustenborder/kafka-connect-spooldir github https://github.com/jcustenborder/kafka-connect-spooldir/issues com.github.jcustenborder.kafka.connect connect-utils-parser net.sourceforge.argparse4j argparse4j 0.7.0 provided com.opencsv opencsv 4.6 commons-beanutils commons-beanutils 1.9.4 io.confluent kafka-connect-avro-converter 5.2.1 provided org.apache.commons commons-compress [1.19,) com.github.jcustenborder.parsers extended-log-format [0.0.2.12, 0.0.2.1000) io.confluent kafka-connect-maven-plugin 0.11.2 hub kafka-connect true https://docs.confluent.io/kafka-connect-spooldir/current/index.html source File Flume csv json Kafka Connect Spooldir Confluent, Inc. https://docs.confluent.io/kafka-connect-spooldir/current/index.html supported by Confluent as part of a Confluent Platform subscription.]]> ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractCleanUpPolicy.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; abstract class AbstractCleanUpPolicy implements Closeable { private static final Logger log = LoggerFactory.getLogger(AbstractCleanUpPolicy.class); private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd"); protected final InputFile inputFile; protected final File errorPath; protected final File finishedPath; protected AbstractCleanUpPolicy(InputFile inputFile, File errorPath, File finishedPath) { this.inputFile = inputFile; this.errorPath = errorPath; this.finishedPath = finishedPath; } public static AbstractCleanUpPolicy create(AbstractSourceConnectorConfig config, InputFile inputFile) throws IOException { final AbstractCleanUpPolicy result; switch (config.cleanupPolicy) { case MOVE: result = new Move(inputFile, config.errorPath, config.finishedPath); break; case MOVEBYDATE: result = new MoveByDate(inputFile, config.errorPath, config.finishedPath); break; case DELETE: result = new Delete(inputFile, config.errorPath, config.finishedPath); break; case NONE: result = new None(inputFile, config.errorPath, config.finishedPath); break; default: throw new UnsupportedOperationException( String.format("%s is not supported", config.cleanupPolicy) ); } return result; } protected boolean createDirectory(File directory) { if (directory.exists()) { return true; } if (!directory.mkdir()) { log.error("Cannot make directory - " + directory.getAbsolutePath()); return false; } if (!directory.setWritable(true)) { log.error("Cannot make directory writable - " + directory.getAbsolutePath()); return false; } return true; } @Override public void close() throws IOException { this.inputFile.close(); } /** * Method is used to handle file cleanup when processing the file has errored. */ public void error() throws IOException { close(); log.error( "Error during processing, moving {} to {}.", this.inputFile, this.errorPath ); this.inputFile.moveToDirectory(this.errorPath); } /** * Method is used to handle file cleanup when processing the file was successful. */ public void success() throws IOException { close(); } static class Move extends AbstractCleanUpPolicy { protected Move(InputFile inputFile, File errorPath, File finishedPath) { super(inputFile, errorPath, finishedPath); } @Override public void success() throws IOException { super.success(); this.inputFile.moveToDirectory(this.finishedPath); } } static class MoveByDate extends AbstractCleanUpPolicy { protected MoveByDate(InputFile inputFile, File errorPath, File finishedPath) { super(inputFile, errorPath, finishedPath); } @Override public void success() throws IOException { super.success(); // Setup directory named as the file created date File subDirectory = new File(this.finishedPath, DATE_FORMATTER.format(this.inputFile.lastModified())); log.trace("Finished path: {}", subDirectory); if (createDirectory(subDirectory)) { this.inputFile.moveToDirectory(subDirectory); } else { this.inputFile.moveToDirectory(this.finishedPath); } } } static class Delete extends AbstractCleanUpPolicy { protected Delete(InputFile inputFile, File errorPath, File finishedPath) { super(inputFile, errorPath, finishedPath); } @Override public void success() throws IOException { super.success(); this.inputFile.delete(); } } static class None extends AbstractCleanUpPolicy { protected None(InputFile inputFile, File errorPath, File finishedPath) { super(inputFile, errorPath, finishedPath); } @Override public void success() throws IOException { super.success(); log.trace("Leaving {}", this.inputFile); } } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSchemaGenerator.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.fasterxml.jackson.core.JsonProcessingException; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import net.sourceforge.argparse4j.ArgumentParsers; import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.AbstractMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; public abstract class AbstractSchemaGenerator { static final String DUMMY_SCHEMA; static final Map DEFAULTS; private static final Logger log = LoggerFactory.getLogger(AbstractSchemaGenerator.class); static { String dummySchema; try { dummySchema = ObjectMapperFactory.INSTANCE.writeValueAsString(SchemaBuilder.struct().build()); } catch (JsonProcessingException e) { dummySchema = null; } DUMMY_SCHEMA = dummySchema; } static { Map defaultSettings = new LinkedHashMap<>(); defaultSettings.put(AbstractSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, ".*"); defaultSettings.put(AbstractSourceConnectorConfig.INPUT_PATH_CONFIG, "/tmp/input"); defaultSettings.put(AbstractSourceConnectorConfig.FINISHED_PATH_CONFIG, "/tmp/finish"); defaultSettings.put(AbstractSourceConnectorConfig.ERROR_PATH_CONFIG, "/tmp/error"); defaultSettings.put(AbstractSpoolDirSourceConnectorConfig.VALUE_SCHEMA_CONF, DUMMY_SCHEMA); defaultSettings.put(AbstractSpoolDirSourceConnectorConfig.KEY_SCHEMA_CONF, DUMMY_SCHEMA); defaultSettings.put(AbstractSourceConnectorConfig.TOPIC_CONF, "dummy"); defaultSettings.put(AbstractSpoolDirSourceConnectorConfig.SCHEMA_GENERATION_ENABLED_CONF, "true"); DEFAULTS = ImmutableMap.copyOf(defaultSettings); } protected CONFIG config; public AbstractSchemaGenerator(Map settings) { Map copySettings = new LinkedHashMap<>(settings); for (Map.Entry kvp : DEFAULTS.entrySet()) { if (!copySettings.containsKey(kvp.getKey())) { copySettings.put(kvp.getKey(), kvp.getValue()); } } this.config = config(copySettings); } public static void main(String... args) throws Exception { ArgumentParser parser = ArgumentParsers.newArgumentParser("CsvSchemaGenerator") .defaultHelp(true) .description("Generate a schema based on a file."); parser.addArgument("-t", "--type") .required(true) .choices("csv", "json") .help("The type of generator to use."); parser.addArgument("-c", "--config") .type(File.class); parser.addArgument("-f", "--file") .type(File.class) .required(true) .help("The data file to generate the schema from."); parser.addArgument("-i", "--id") .nargs("*") .help("Field(s) to use as an identifier."); parser.addArgument("-o", "--output") .type(File.class) .help("Output location to write the configuration to. Stdout is default."); Namespace ns = null; try { ns = parser.parseArgs(args); } catch (ArgumentParserException ex) { parser.handleError(ex); System.exit(1); } File inputFile = ns.get("file"); List ids = ns.getList("id"); if (null == ids) { ids = ImmutableList.of(); } Map settings = new LinkedHashMap<>(); File inputPropertiesFile = ns.get("config"); if (null != inputPropertiesFile) { Properties inputProperties = new Properties(); try (FileInputStream inputStream = new FileInputStream(inputPropertiesFile)) { inputProperties.load(inputStream); } for (String s : inputProperties.stringPropertyNames()) { Object v = inputProperties.getProperty(s); settings.put(s, v); } } final AbstractSchemaGenerator generator; final String type = ns.getString("type"); if ("csv".equalsIgnoreCase(type)) { generator = new CsvSchemaGenerator(settings); } else if ("json".equalsIgnoreCase(type)) { generator = new JsonSchemaGenerator(settings); } else { throw new UnsupportedOperationException( String.format("'%s' is not a supported schema generator type", type) ); } Map.Entry kvp = generator.generate(inputFile, ids); Properties properties = new Properties(); properties.putAll(settings); properties.setProperty(AbstractSpoolDirSourceConnectorConfig.KEY_SCHEMA_CONF, ObjectMapperFactory.INSTANCE.writeValueAsString(kvp.getKey())); properties.setProperty(AbstractSpoolDirSourceConnectorConfig.VALUE_SCHEMA_CONF, ObjectMapperFactory.INSTANCE.writeValueAsString(kvp.getValue())); String output = ns.getString("output"); final String comment = "Configuration was dynamically generated. Please verify before submitting."; if (Strings.isNullOrEmpty(output)) { properties.store(System.out, comment); } else { try (FileOutputStream outputStream = new FileOutputStream(output)) { properties.store(outputStream, comment); } } } protected abstract CONFIG config(Map settings); protected abstract Map determineFieldTypes(InputStream inputStream) throws IOException; void addField(SchemaBuilder builder, String name, Schema.Type schemaType) { log.trace("addField() - name = {} schemaType = {}", name, schemaType); builder.field( name, SchemaBuilder.type(schemaType).optional().build() ); } public Map.Entry generate(File inputFile, List keyFields) throws IOException { log.trace("generate() - inputFile = '{}', keyFields = {}", inputFile, keyFields); final Map fieldTypes; log.info("Determining fields from {}", inputFile); try (InputStream inputStream = new FileInputStream(inputFile)) { fieldTypes = determineFieldTypes(inputStream); } log.trace("generate() - Building key schema."); SchemaBuilder keySchemaBuilder = SchemaBuilder.struct() .name(this.config.schemaGenerationKeyName); for (String keyFieldName : keyFields) { log.trace("generate() - Adding keyFieldName field '{}'", keyFieldName); if (fieldTypes.containsKey(keyFieldName)) { Schema.Type schemaType = fieldTypes.get(keyFieldName); addField(keySchemaBuilder, keyFieldName, schemaType); } else { log.warn("Key field '{}' is not in the data.", keyFieldName); } } log.trace("generate() - Building value schema."); SchemaBuilder valueSchemaBuilder = SchemaBuilder.struct() .name(this.config.schemaGenerationValueName); for (Map.Entry kvp : fieldTypes.entrySet()) { addField(valueSchemaBuilder, kvp.getKey(), kvp.getValue()); } return new AbstractMap.SimpleEntry<>(keySchemaBuilder.build(), valueSchemaBuilder.build()); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.VersionUtil; import org.apache.kafka.connect.source.SourceConnector; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; public abstract class AbstractSourceConnector extends SourceConnector { protected Map settings; protected CONF config; protected abstract CONF config(Map settings); @Override public void start(Map settings) { this.config = config(settings); this.settings = settings; } @Override public List> taskConfigs(int taskCount) { List> result = new ArrayList<>(); for (int i = 0; i < taskCount; i++) { Map taskConfig = new LinkedHashMap<>(this.settings); taskConfig.put(AbstractSourceConnectorConfig.TASK_INDEX_CONF, Integer.toString(i)); taskConfig.put(AbstractSourceConnectorConfig.TASK_COUNT_CONF, Integer.toString(taskCount)); result.add(taskConfig); } return result; } @Override public void stop() { } @Override public String version() { return VersionUtil.version(this.getClass()); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils; import com.github.jcustenborder.kafka.connect.utils.config.ValidEnum; import com.github.jcustenborder.kafka.connect.utils.config.ValidPattern; import com.github.jcustenborder.kafka.connect.utils.config.recommenders.Recommenders; import com.github.jcustenborder.kafka.connect.utils.config.validators.Validators; import com.github.jcustenborder.kafka.connect.utils.config.validators.filesystem.ValidDirectoryWritable; import com.google.common.collect.ImmutableList; import com.google.common.io.PatternFilenameFilter; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; import java.io.File; import java.util.List; import java.util.Map; import java.util.regex.Pattern; public abstract class AbstractSourceConnectorConfig extends AbstractConfig { public static final String FINISHED_PATH_CONFIG = "finished.path"; public static final String ERROR_PATH_CONFIG = "error.path"; public static final String INPUT_FILE_PATTERN_CONF = "input.file.pattern"; public static final String HALT_ON_ERROR_CONF = "halt.on.error"; public static final String FILE_MINIMUM_AGE_MS_CONF = "file.minimum.age.ms"; public static final String FILE_SORT_ATTRIBUTES_CONF = "files.sort.attributes"; public static final String INPUT_PATH_WALK_RECURSIVELY = "input.path.walk.recursively"; public static final boolean INPUT_PATH_WALK_RECURSIVELY_DEFAULT = false; static final String INPUT_PATH_WALK_RECURSIVELY_DOC = "If enabled, any sub-directories dropped under `input.path` will be recursively walked looking for files matching the configured `input.file.pattern`. After processing is complete the discovered sub directory structure (as well as files within them) will handled according to the configured `cleanup.policy` (i.e. moved or deleted etc). For each discovered file, the walked sub-directory path will be set as a header named `file.relative.path`"; public static final String CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH = "cleanup.policy.maintain.relative.path"; static final boolean CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH_DEFAULT = false; static final String CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH_DOC = "If `" + INPUT_PATH_WALK_RECURSIVELY + "` is enabled in combination with this flag being `true`, the walked sub-directories which contained files will be retained as-is under the `input.path`. The actual files within the sub-directories will moved (with a copy of the sub-dir structure) or deleted as per the `cleanup.policy` defined, but the parent sub-directory structure will remain."; public static final String PROCESSING_FILE_EXTENSION_CONF = "processing.file.extension"; //RecordProcessorConfig public static final String BATCH_SIZE_CONF = "batch.size"; public static final String PROCESSING_FILE_EXTENSION_DEFAULT = ".PROCESSING"; public static final String TOPIC_CONF = "topic"; public static final String EMPTY_POLL_WAIT_MS_CONF = "empty.poll.wait.ms"; public static final String CLEANUP_POLICY_CONF = "cleanup.policy"; public static final String CLEANUP_POLICY_DOC = "Determines how the connector should cleanup the " + "files that have been successfully processed. NONE leaves the files in place which could " + "cause them to be reprocessed if the connector is restarted. DELETE removes the file from the " + "filesystem. MOVE will move the file to a finished directory. MOVEBYDATE will move the file to " + "a finished directory with subdirectories by date"; public static final String GROUP_FILESYSTEM = "File System"; public static final String GROUP_GENERAL = "General"; //DirectoryMonitorConfig //PollingDirectoryMonitorConfig public static final String INPUT_PATH_CONFIG = "input.path"; public static final String TIMESTAMP_MODE_CONF = "timestamp.mode"; public static final String GROUP_TIMESTAMP = "Timestamps"; static final String BATCH_SIZE_DOC = "The number of records that should be returned with each batch."; static final int BATCH_SIZE_DEFAULT = 1000; static final String TOPIC_DOC = "The Kafka topic to write the data to."; static final String INPUT_PATH_DOC = "The directory to read files that will be processed. This directory must exist and be writable by the user running Kafka Connect."; static final String FINISHED_PATH_DOC = "The directory to place files that have been successfully processed. This directory must exist and be writable by the user running Kafka Connect."; static final String ERROR_PATH_DOC = "The directory to place files in which have error(s). This directory must exist and be writable by the user running Kafka Connect."; static final String INPUT_FILE_PATTERN_DOC = "Regular expression to check input file names against. This expression " + "must match the entire filename. The equivalent of Matcher.matches()."; static final String HALT_ON_ERROR_DOC = "Should the task halt when it encounters an error or continue to the next file."; static final String FILE_MINIMUM_AGE_MS_DOC = "The amount of time in milliseconds after the file was last written to before the file can be processed."; static final String PROCESSING_FILE_EXTENSION_DOC = "Before a file is processed, a flag is created in its directory to indicate the file is being handled. " + "The flag file has the same name as the file, but with this property appended as a suffix."; static final String EMPTY_POLL_WAIT_MS_DOC = "The amount of time to wait if a poll returns an empty list of records."; static final String TIMESTAMP_FIELD_DOC = "The field in the value schema that will contain the parsed timestamp for the record. " + "This field cannot be marked as optional and must be a " + "[Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html)"; static final String TIMESTAMP_MODE_DOC = "Determines how the connector will set the timestamp for the " + "[ConnectRecord](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/connector/ConnectRecord.html#timestamp()). " + "If set to `Field` then the timestamp will be read from a field in the value. This field cannot be optional and must be " + "a [Timestamp](https://kafka.apache.org/0102/javadoc/org/apache/kafka/connect/data/Schema.html). Specify the field " + " in `" + AbstractSpoolDirSourceConnectorConfig.TIMESTAMP_FIELD_CONF + "`. " + "If set to `FILE_TIME` then " + "the last modified time of the file will be used. If set to `PROCESS_TIME` the time the record is read will be used."; static final String FILE_SORT_ATTRIBUTES_DOC = "The attributes each file will use to determine the sort order. " + "`Name` is name of the file. `Length` is the length of the file preferring larger files first. `LastModified` is " + "the LastModified attribute of the file preferring older files first."; public static final String TASK_INDEX_CONF = "task.index"; static final String TASK_INDEX_DOC = "Internal setting to the connector used to instruct a " + "task on which files to select. The connector will override this setting."; public static final String TASK_COUNT_CONF = "task.count"; static final String TASK_COUNT_DOC = "Internal setting to the connector used to instruct a " + "task on which files to select. The connector will override this setting."; public static final String TASK_PARTITIONER_CONF = "task.partitioner"; static final String TASK_PARTITIONER_DOC = "The task partitioner implementation is used when the " + "connector is configured to use more than one task. This is used by each task to identify " + "which files will be processed by that task. This ensures that each file is only assigned to " + "one task."; public static final String FILE_BUFFER_SIZE_CONF = "file.buffer.size.bytes"; static final String FILE_BUFFER_SIZE_DOC = "The size of buffer for the BufferedInputStream that will be used to " + "interact with the file system."; public final File inputPath; public final File finishedPath; public final File errorPath; public final boolean haltOnError; public final long minimumFileAgeMS; public final int batchSize; public final String topic; public final long emptyPollWaitMs; public final String processingFileExtension; public final TimestampMode timestampMode; public final CleanupPolicy cleanupPolicy; public final PatternFilenameFilter inputFilenameFilter; public final List fileSortAttributes; public final int taskIndex; public final int taskCount; public final TaskPartitioner taskPartitioner; public final boolean bufferedInputStream; public final int fileBufferSizeBytes; public final boolean inputPathWalkRecursively; public final boolean inputPathWalkRecursivelyRetainSubDirs; public final boolean finishedPathRequired() { boolean result; switch (this.cleanupPolicy) { case MOVE: case MOVEBYDATE: result = true; break; default: result = false; } return result; } public AbstractSourceConnectorConfig(ConfigDef definition, Map originals, boolean bufferedInputStream) { super(definition, originals); this.bufferedInputStream = bufferedInputStream; this.inputPath = ConfigUtils.getAbsoluteFile(this, INPUT_PATH_CONFIG); this.cleanupPolicy = ConfigUtils.getEnum(CleanupPolicy.class, this, CLEANUP_POLICY_CONF); if (finishedPathRequired()) { this.finishedPath = ConfigUtils.getAbsoluteFile(this, FINISHED_PATH_CONFIG); } else { this.finishedPath = null; } this.errorPath = ConfigUtils.getAbsoluteFile(this, ERROR_PATH_CONFIG); this.haltOnError = this.getBoolean(HALT_ON_ERROR_CONF); this.minimumFileAgeMS = this.getLong(FILE_MINIMUM_AGE_MS_CONF); this.batchSize = this.getInt(BATCH_SIZE_CONF); this.topic = this.getString(TOPIC_CONF); this.emptyPollWaitMs = this.getLong(EMPTY_POLL_WAIT_MS_CONF); this.processingFileExtension = this.getString(PROCESSING_FILE_EXTENSION_CONF); this.timestampMode = ConfigUtils.getEnum(TimestampMode.class, this, TIMESTAMP_MODE_CONF); final String inputPatternText = this.getString(INPUT_FILE_PATTERN_CONF); final Pattern inputPattern = Pattern.compile(inputPatternText); this.inputFilenameFilter = new PatternFilenameFilter(inputPattern); this.fileSortAttributes = ConfigUtils.getEnums(FileAttribute.class, this, FILE_SORT_ATTRIBUTES_CONF); this.taskIndex = getInt(TASK_INDEX_CONF); this.taskCount = getInt(TASK_COUNT_CONF); this.taskPartitioner = ConfigUtils.getEnum(TaskPartitioner.class, this, TASK_PARTITIONER_CONF); this.inputPathWalkRecursively = this.getBoolean(INPUT_PATH_WALK_RECURSIVELY); this.inputPathWalkRecursivelyRetainSubDirs = this.getBoolean(CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH); if (bufferedInputStream) { this.fileBufferSizeBytes = getInt(FILE_BUFFER_SIZE_CONF); } else { this.fileBufferSizeBytes = 0; } } protected static ConfigDef config(boolean bufferedInputStream) { final ConfigDef result = new ConfigDef() .define( ConfigKeyBuilder.of(TOPIC_CONF, ConfigDef.Type.STRING) .documentation(TOPIC_DOC) .group(GROUP_GENERAL) .importance(ConfigDef.Importance.HIGH) .build() ).define( ConfigKeyBuilder.of(BATCH_SIZE_CONF, ConfigDef.Type.INT) .documentation(BATCH_SIZE_DOC) .importance(ConfigDef.Importance.LOW) .defaultValue(BATCH_SIZE_DEFAULT) .group(GROUP_GENERAL) .build() ).define( ConfigKeyBuilder.of(EMPTY_POLL_WAIT_MS_CONF, ConfigDef.Type.LONG) .documentation(EMPTY_POLL_WAIT_MS_DOC) .importance(ConfigDef.Importance.LOW) .defaultValue(500L) .validator(ConfigDef.Range.between(1L, Long.MAX_VALUE)) .group(GROUP_GENERAL) .build() ) .define( ConfigKeyBuilder.of(CLEANUP_POLICY_CONF, ConfigDef.Type.STRING) .documentation(CLEANUP_POLICY_DOC) .importance(ConfigDef.Importance.MEDIUM) .validator(ValidEnum.of(CleanupPolicy.class)) .defaultValue(CleanupPolicy.MOVE.toString()) .group(GROUP_FILESYSTEM) .build() ) .define( ConfigKeyBuilder.of(INPUT_PATH_CONFIG, ConfigDef.Type.STRING) .documentation(INPUT_PATH_DOC) .importance(ConfigDef.Importance.HIGH) .validator(ValidDirectoryWritable.of()) .group(GROUP_FILESYSTEM) .build() ).define( ConfigKeyBuilder.of(FINISHED_PATH_CONFIG, ConfigDef.Type.STRING) .documentation(FINISHED_PATH_DOC) .importance(ConfigDef.Importance.HIGH) .defaultValue("") .recommender( Recommenders.visibleIf(CLEANUP_POLICY_CONF, CleanupPolicy.MOVE.toString()) ) .group(GROUP_FILESYSTEM) .build() ).define( ConfigKeyBuilder.of(ERROR_PATH_CONFIG, ConfigDef.Type.STRING) .documentation(ERROR_PATH_DOC) .importance(ConfigDef.Importance.HIGH) .validator(ValidDirectoryWritable.of()) .group(GROUP_FILESYSTEM) .build() ).define( ConfigKeyBuilder.of(INPUT_FILE_PATTERN_CONF, ConfigDef.Type.STRING) .documentation(INPUT_FILE_PATTERN_DOC) .importance(ConfigDef.Importance.HIGH) .group(GROUP_FILESYSTEM) .build() ).define( ConfigKeyBuilder.of(HALT_ON_ERROR_CONF, ConfigDef.Type.BOOLEAN) .documentation(HALT_ON_ERROR_DOC) .importance(ConfigDef.Importance.HIGH) .defaultValue(true) .group(GROUP_FILESYSTEM) .build() ).define( ConfigKeyBuilder.of(FILE_MINIMUM_AGE_MS_CONF, ConfigDef.Type.LONG) .documentation(FILE_MINIMUM_AGE_MS_DOC) .importance(ConfigDef.Importance.LOW) .group(GROUP_FILESYSTEM) .defaultValue(0L) .validator(ConfigDef.Range.atLeast(0L)) .build() ).define( ConfigKeyBuilder.of(PROCESSING_FILE_EXTENSION_CONF, ConfigDef.Type.STRING) .documentation(PROCESSING_FILE_EXTENSION_DOC) .importance(ConfigDef.Importance.LOW) .validator(ValidDirectoryWritable.of()) .group(GROUP_FILESYSTEM) .defaultValue(PROCESSING_FILE_EXTENSION_DEFAULT) .validator(ValidPattern.of("^.*\\..+$")) .build() ).define( ConfigKeyBuilder.of(TIMESTAMP_MODE_CONF, ConfigDef.Type.STRING) .documentation(TIMESTAMP_MODE_DOC) .importance(ConfigDef.Importance.MEDIUM) .group(GROUP_TIMESTAMP) .defaultValue(TimestampMode.PROCESS_TIME.toString()) .validator(ValidEnum.of(TimestampMode.class)) .build() ).define( ConfigKeyBuilder.of(FILE_SORT_ATTRIBUTES_CONF, ConfigDef.Type.LIST) .documentation(FILE_SORT_ATTRIBUTES_DOC) .importance(ConfigDef.Importance.LOW) .validator(Validators.validEnum(FileAttribute.class)) .group(GROUP_FILESYSTEM) .defaultValue(ImmutableList.of(FileAttribute.NameAsc.name())) .build() ).define( ConfigKeyBuilder.of(TASK_INDEX_CONF, ConfigDef.Type.INT) .documentation(TASK_INDEX_DOC) .importance(ConfigDef.Importance.LOW) .validator(ConfigDef.Range.atLeast(0)) .group(GROUP_GENERAL) .defaultValue(0) .build() ).define( ConfigKeyBuilder.of(TASK_COUNT_CONF, ConfigDef.Type.INT) .documentation(TASK_COUNT_DOC) .importance(ConfigDef.Importance.LOW) .validator(ConfigDef.Range.atLeast(1)) .group(GROUP_GENERAL) .defaultValue(1) .build() ).define( ConfigKeyBuilder.of(TASK_PARTITIONER_CONF, ConfigDef.Type.STRING) .documentation(TASK_PARTITIONER_DOC) .importance(ConfigDef.Importance.MEDIUM) .validator(Validators.validEnum(TaskPartitioner.class)) .defaultValue(TaskPartitioner.ByName.toString()) .group(GROUP_FILESYSTEM) .build() ).define( ConfigKeyBuilder.of(INPUT_PATH_WALK_RECURSIVELY, ConfigDef.Type.BOOLEAN) .documentation(INPUT_PATH_WALK_RECURSIVELY_DOC) .importance(ConfigDef.Importance.LOW) .defaultValue(INPUT_PATH_WALK_RECURSIVELY_DEFAULT) .group(GROUP_FILESYSTEM) .build() ).define( ConfigKeyBuilder.of(CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH, ConfigDef.Type.BOOLEAN) .documentation(CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH_DOC) .importance(ConfigDef.Importance.LOW) .defaultValue(CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH_DEFAULT) .group(GROUP_FILESYSTEM) .build() ); if (bufferedInputStream) { result.define( ConfigKeyBuilder.of(FILE_BUFFER_SIZE_CONF, ConfigDef.Type.INT) .documentation(FILE_BUFFER_SIZE_DOC) .importance(ConfigDef.Importance.LOW) .validator(ConfigDef.Range.atLeast(1)) .defaultValue(128 * 1024) .group(GROUP_FILESYSTEM) .build() ); } return result; } public enum TimestampMode { FIELD, FILE_TIME, PROCESS_TIME } public enum CleanupPolicy { NONE, DELETE, MOVE, MOVEBYDATE } public enum FileAttribute { NameAsc, NameDesc, LengthAsc, LengthDesc, LastModifiedAsc, LastModifiedDesc } public enum TaskPartitioner { ByName } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.VersionUtil; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; public abstract class AbstractSourceTask extends SourceTask { private static final Logger log = LoggerFactory.getLogger(AbstractSourceTask.class); protected Map sourcePartition; protected CONF config; private final Stopwatch processingTime = Stopwatch.createUnstarted(); protected InputFile inputFile; private boolean hasRecords = false; private static void checkDirectory(String key, File directoryPath) { if (log.isInfoEnabled()) { log.info("Checking if directory {} '{}' exists.", key, directoryPath ); } String errorMessage = String.format( "Directory for '%s' '%s' does not exist ", key, directoryPath ); if (!directoryPath.isDirectory()) { throw new ConnectException( errorMessage, new FileNotFoundException(directoryPath.getAbsolutePath()) ); } if (log.isInfoEnabled()) { log.info("Checking to ensure {} '{}' is writable ", key, directoryPath); } errorMessage = String.format( "Directory for '%s' '%s' it not writable.", key, directoryPath ); File temporaryFile = null; try { temporaryFile = File.createTempFile(".permission", ".testing", directoryPath); } catch (IOException ex) { throw new ConnectException( errorMessage, ex ); } finally { try { if (null != temporaryFile && temporaryFile.exists()) { Preconditions.checkState(temporaryFile.delete(), "Unable to delete temp file in %s", directoryPath); } } catch (Exception ex) { if (log.isWarnEnabled()) { log.warn("Exception thrown while deleting {}.", temporaryFile, ex); } } } } protected abstract CONF config(Map settings); protected abstract void configure(InputFile inputFile, Long lastOffset) throws IOException; protected abstract List process() throws IOException; protected abstract long recordOffset(); @Override public void start(Map settings) { this.config = config(settings); checkDirectory(AbstractSourceConnectorConfig.INPUT_PATH_CONFIG, this.config.inputPath); checkDirectory(AbstractSourceConnectorConfig.ERROR_PATH_CONFIG, this.config.errorPath); if (this.config.finishedPathRequired()) { checkDirectory(AbstractSourceConnectorConfig.FINISHED_PATH_CONFIG, this.config.finishedPath); } this.inputFileDequeue = new InputFileDequeue(this.config); } @Override public void stop() { log.info("Stopping task."); try { if (null != this.inputFile) { this.inputFile.close(); } if (null != this.cleanUpPolicy) { this.cleanUpPolicy.close(); } } catch (IOException ex) { log.error("Exception thrown while closing {}", this.inputFile); } } @Override public String version() { return VersionUtil.version(this.getClass()); } InputFileDequeue inputFileDequeue; int emptyCount = 0; long recordCount; @Override public List poll() throws InterruptedException { log.trace("poll()"); List results = read(); if (results.isEmpty()) { emptyCount++; if (emptyCount > 1) { log.trace("read() returned empty list. Sleeping {} ms.", this.config.emptyPollWaitMs); Thread.sleep(this.config.emptyPollWaitMs); } return null; } emptyCount = 0; log.trace("read() returning {} result(s)", results.size()); return results; } // /** * Calculates the byte count in a human readable form. Special thanks to * https://stackoverflow.com/questions/3758606/how-to-convert-byte-size-into-human-readable-format-in-java * * @param bytes * @param si * @return */ public static String humanReadableByteCount(long bytes, boolean si) { final int unit = si ? 1000 : 1024; if (bytes < unit) return bytes + " B"; int exp = (int) (Math.log(bytes) / Math.log(unit)); String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); } private void recordProcessingTime() { final long secondsElapsed = processingTime.elapsed(TimeUnit.SECONDS); final long bytesPerSecond; if (0L == secondsElapsed || 0L == this.inputFile.length()) { bytesPerSecond = 0L; } else { bytesPerSecond = this.inputFile.length() / secondsElapsed; } if (bytesPerSecond > 0) { log.info( "Finished processing {} record(s) in {} second(s). Processing speed {} per second.", this.recordCount, secondsElapsed, humanReadableByteCount(bytesPerSecond, false) ); } else { log.info( "Finished processing {} record(s) in {} second(s).", this.recordCount, secondsElapsed ); } } AbstractCleanUpPolicy cleanUpPolicy; public List read() { try { if (!hasRecords) { if (null != this.inputFile) { recordProcessingTime(); this.inputFile.close(); this.cleanUpPolicy.success(); this.inputFile = null; } log.trace("read() - polling for next file."); InputFile nextFile = this.inputFileDequeue.poll(); log.trace("read() - nextFile = '{}'", nextFile); if (null == nextFile) { log.trace("read() - No next file found."); return new ArrayList<>(); } this.inputFile = nextFile; try { this.sourcePartition = ImmutableMap.of( "fileName", this.inputFile.getName() ); log.info("Opening {}", this.inputFile); Long lastOffset = null; log.trace("looking up offset for {}", this.sourcePartition); Map offset = this.context.offsetStorageReader().offset(this.sourcePartition); if (null != offset && !offset.isEmpty()) { Number number = (Number) offset.get("offset"); lastOffset = number.longValue(); } this.cleanUpPolicy = AbstractCleanUpPolicy.create(this.config, this.inputFile); this.recordCount = 0; log.trace("read() - calling configure(lastOffset={})", lastOffset); configure(this.inputFile, lastOffset); } catch (Exception ex) { throw new ConnectException(ex); } processingTime.reset(); processingTime.start(); } List records = process(); this.hasRecords = !records.isEmpty(); return records; } catch (Exception ex) { long recordOffset; try { recordOffset = recordOffset(); } catch (Exception e) { log.error("Exception thrown while calling recordOffset()", e); recordOffset = -1; } log.error("Exception encountered processing line {} of {}.", recordOffset, this.inputFile, ex); try { this.cleanUpPolicy.error(); } catch (IOException e) { log.warn("Exception while while closing cleanup policy", ex); } if (this.config.haltOnError) { throw new ConnectException(ex); } else { return new ArrayList<>(); } } } protected Map offset() { return ImmutableMap.of( "offset", recordOffset() ); } protected SourceRecord record( SchemaAndValue key, SchemaAndValue value, Long timestamp) { Map sourceOffset = offset(); SourceRecord result = new SourceRecord( this.sourcePartition, sourceOffset, this.config.topic, null, null != key ? key.schema() : null, null != key ? key.value() : null, value.schema(), value.value(), timestamp, this.inputFile.metadata().headers(recordOffset()) ); return result; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSpoolDirSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.VersionUtil; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import com.google.common.base.Preconditions; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.errors.DataException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; public abstract class AbstractSpoolDirSourceConnector extends AbstractSourceConnector { private static final Logger log = LoggerFactory.getLogger(AbstractSpoolDirSourceConnector.class); protected abstract AbstractSchemaGenerator generator(Map settings); @Override public String version() { return VersionUtil.version(this.getClass()); } @Override public void start(final Map input) { super.start(input); this.config = config(input); final Map settings = new LinkedHashMap<>(input); if (this.config.schemasRequired() && (null == this.config.valueSchema || null == this.config.keySchema)) { log.info("Key or Value schema was not defined. Running schema generator."); AbstractSchemaGenerator generator = generator(settings); try { List inputFiles = Arrays.stream(this.config.inputPath.listFiles(this.config.inputFilenameFilter)) .limit(5) .collect(Collectors.toList()); Preconditions.checkState( !inputFiles.isEmpty(), "Could not find any input file(s) to infer schema from." ); Map> schemas = new HashMap<>(); Multimap schemaToFiles = HashMultimap.create(); for (File inputFile : inputFiles) { Map.Entry schemaEntry = generator.generate(inputFile, this.config.keyFields); String schema = ObjectMapperFactory.INSTANCE.writeValueAsString(schemaEntry.getValue()); schemaToFiles.put(schema, inputFile); schemas.put(schema, schemaEntry); } Map> schemaToFilesMap = schemaToFiles.asMap(); if (1 != schemaToFilesMap.keySet().size()) { StringBuilder builder = new StringBuilder(); builder.append("More than one schema was found for the input pattern.\n"); for (String schema : schemaToFilesMap.keySet()) { builder.append("Schema: "); builder.append(schema); builder.append("\n"); for (File f : schemaToFilesMap.get(schema)) { builder.append(" "); builder.append(f); builder.append("\n"); } } throw new DataException(builder.toString()); } Map.Entry schemaPair = null; for (Map.Entry s : schemas.values()) { schemaPair = s; break; } if (null == schemaPair) { throw new DataException("Schema could not be generated."); } final String keySchema = ObjectMapperFactory.INSTANCE.writeValueAsString(schemaPair.getKey()); log.info("Setting {} to {}", AbstractSpoolDirSourceConnectorConfig.KEY_SCHEMA_CONF, keySchema); final String valueSchema = ObjectMapperFactory.INSTANCE.writeValueAsString(schemaPair.getValue()); log.info("Setting {} to {}", AbstractSpoolDirSourceConnectorConfig.VALUE_SCHEMA_CONF, valueSchema); settings.put(AbstractSpoolDirSourceConnectorConfig.KEY_SCHEMA_CONF, keySchema); settings.put(AbstractSpoolDirSourceConnectorConfig.VALUE_SCHEMA_CONF, valueSchema); } catch (IOException e) { throw new ConnectException("Exception thrown while generating schema", e); } this.settings = settings; } this.settings = settings; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSpoolDirSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.fasterxml.jackson.core.JsonProcessingException; import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; import com.github.jcustenborder.kafka.connect.utils.config.recommenders.Recommenders; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Type; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Timestamp; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.errors.DataException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TimeZone; @SuppressWarnings("WeakerAccess") public abstract class AbstractSpoolDirSourceConnectorConfig extends AbstractSourceConnectorConfig { public static final String TIMESTAMP_FIELD_CONF = "timestamp.field"; public static final String KEY_SCHEMA_CONF = "key.schema"; public static final String VALUE_SCHEMA_CONF = "value.schema"; public static final String PARSER_TIMESTAMP_DATE_FORMATS_CONF = "parser.timestamp.date.formats"; public static final String PARSER_TIMESTAMP_TIMEZONE_CONF = "parser.timestamp.timezone"; public static final String SCHEMA_GENERATION_KEY_FIELDS_CONF = "schema.generation.key.fields"; public static final String SCHEMA_GENERATION_KEY_NAME_CONF = "schema.generation.key.name"; public static final String SCHEMA_GENERATION_VALUE_NAME_CONF = "schema.generation.value.name"; public static final String SCHEMA_GENERATION_ENABLED_CONF = "schema.generation.enabled"; public static final String GROUP_SCHEMA_GENERATION = "Schema Generation"; public static final String GROUP_SCHEMA = "Schema"; static final String KEY_SCHEMA_DOC = "The schema for the key written to Kafka."; static final String VALUE_SCHEMA_DOC = "The schema for the value written to Kafka."; static final String PARSER_TIMESTAMP_DATE_FORMATS_DOC = "The date formats that are expected in the file. This is a list " + "of strings that will be used to parse the date fields in order. The most accurate date format should be the first " + "in the list. Take a look at the Java documentation for more info. " + "https://docs.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html"; static final List PARSER_TIMESTAMP_DATE_FORMATS_DEFAULT = Arrays.asList("yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd' 'HH:mm:ss"); static final String PARSER_TIMESTAMP_TIMEZONE_DOC = "The timezone that all of the dates will be parsed with."; static final String PARSER_TIMESTAMP_TIMEZONE_DEFAULT = "UTC"; static final String SCHEMA_GENERATION_KEY_FIELDS_DOC = "The field(s) to use to build a key schema. This is only used during schema generation."; static final String SCHEMA_GENERATION_KEY_NAME_DOC = "The name of the generated key schema."; static final String SCHEMA_GENERATION_VALUE_NAME_DOC = "The name of the generated value schema."; static final String SCHEMA_GENERATION_ENABLED_DOC = "Flag to determine if schemas should be dynamically generated. If set " + " to true, `" + KEY_SCHEMA_CONF + "` and `" + VALUE_SCHEMA_CONF + "` can be omitted, but `" + SCHEMA_GENERATION_KEY_NAME_CONF + "` " + "and `" + SCHEMA_GENERATION_VALUE_NAME_CONF + "` must be set."; private static final Logger log = LoggerFactory.getLogger(AbstractSpoolDirSourceConnectorConfig.class); public final Schema keySchema; public final Schema valueSchema; public final SimpleDateFormat[] parserTimestampDateFormats; public final TimeZone parserTimestampTimezone; public final String timestampField; public final List keyFields; public final boolean schemaGenerationEnabled; public final String schemaGenerationKeyName; public final String schemaGenerationValueName; public AbstractSpoolDirSourceConnectorConfig(final boolean isTask, boolean bufferedInputStream, ConfigDef configDef, Map settings) { super(configDef, settings, bufferedInputStream); this.keyFields = this.getList(SCHEMA_GENERATION_KEY_FIELDS_CONF); this.schemaGenerationEnabled = this.getBoolean(SCHEMA_GENERATION_ENABLED_CONF); this.schemaGenerationKeyName = this.getString(SCHEMA_GENERATION_KEY_NAME_CONF); this.schemaGenerationValueName = this.getString(SCHEMA_GENERATION_VALUE_NAME_CONF); String timestampTimezone = this.getString(PARSER_TIMESTAMP_TIMEZONE_CONF); this.parserTimestampTimezone = TimeZone.getTimeZone(timestampTimezone); List results = new ArrayList<>(); List formats = this.getList(PARSER_TIMESTAMP_DATE_FORMATS_CONF); for (String s : formats) { SimpleDateFormat dateFormat = new SimpleDateFormat(s); dateFormat.setTimeZone(this.parserTimestampTimezone); results.add(dateFormat); } this.parserTimestampDateFormats = results.toArray(new SimpleDateFormat[results.size()]); this.keySchema = readSchema(KEY_SCHEMA_CONF); this.valueSchema = readSchema(VALUE_SCHEMA_CONF); if (!this.schemaGenerationEnabled) { Preconditions.checkNotNull( this.keySchema, "'%s' must be set if '%s' = false.", KEY_SCHEMA_CONF, SCHEMA_GENERATION_ENABLED_CONF ); Preconditions.checkNotNull( this.valueSchema, "'%s' must be set if '%s' = false.", VALUE_SCHEMA_CONF, SCHEMA_GENERATION_ENABLED_CONF ); } else { Preconditions.checkState( !Strings.isNullOrEmpty(this.schemaGenerationKeyName), "'%s' must be set if '%s' = true.", SCHEMA_GENERATION_KEY_NAME_CONF, SCHEMA_GENERATION_ENABLED_CONF ); Preconditions.checkState( !Strings.isNullOrEmpty(this.schemaGenerationValueName), "'%s' must be set if '%s' = true.", SCHEMA_GENERATION_VALUE_NAME_CONF, SCHEMA_GENERATION_ENABLED_CONF ); } if (TimestampMode.FIELD == this.timestampMode) { this.timestampField = this.getString(TIMESTAMP_FIELD_CONF); if (Strings.isNullOrEmpty(this.timestampField)) { throw new ConnectException( String.format( "When `%s` is set to `%s`, `%s` must be set to a timestamp field. Cannot be null or empty.", TIMESTAMP_MODE_CONF, TimestampMode.FIELD, TIMESTAMP_FIELD_CONF ) ); } log.trace("ctor() - Looking for timestamp field '{}'", this.timestampField); Field timestampField = this.valueSchema.field(this.timestampField); if (null == timestampField || timestampField.schema().isOptional() || !Timestamp.LOGICAL_NAME.equals(timestampField.schema().name())) { String example; try { example = ObjectMapperFactory.INSTANCE.writeValueAsString(Timestamp.SCHEMA); } catch (JsonProcessingException e) { example = null; } log.trace("ctor() - example: {}", example); throw new ConnectException( String.format( "Field '%s' must be present and set to a timestamp and cannot be optional. Example %s", this.timestampField, example ) ); } } else { this.timestampField = null; } if (schemasRequired() && (isTask && null == this.valueSchema)) { throw new DataException( String.format("'%s' must be set to a valid schema.", VALUE_SCHEMA_CONF) ); } } protected static ConfigDef config(boolean bufferedInputStream) { ConfigDef.Recommender schemaRecommender = new ConfigDef.Recommender() { @Override public List validValues(String key, Map settings) { return ImmutableList.of(); } @Override public boolean visible(String key, Map settings) { boolean schemaGenerationEnabled = (boolean) settings.get(SCHEMA_GENERATION_ENABLED_CONF); if (KEY_SCHEMA_CONF.endsWith(key)) { return !schemaGenerationEnabled; } if (VALUE_SCHEMA_CONF.endsWith(key)) { return !schemaGenerationEnabled; } if (SCHEMA_GENERATION_KEY_NAME_CONF.endsWith(key)) { return schemaGenerationEnabled; } if (SCHEMA_GENERATION_VALUE_NAME_CONF.endsWith(key)) { return schemaGenerationEnabled; } if (SCHEMA_GENERATION_KEY_FIELDS_CONF.endsWith(key)) { return schemaGenerationEnabled; } return true; } }; return AbstractSourceConnectorConfig.config(bufferedInputStream) .define( ConfigKeyBuilder.of(KEY_SCHEMA_CONF, Type.STRING) .documentation(KEY_SCHEMA_DOC) .importance(ConfigDef.Importance.HIGH) .group(GROUP_SCHEMA) .defaultValue("") .width(ConfigDef.Width.LONG) .build() ).define( ConfigKeyBuilder.of(VALUE_SCHEMA_CONF, Type.STRING) .documentation(VALUE_SCHEMA_DOC) .importance(ConfigDef.Importance.HIGH) .group(GROUP_SCHEMA) .defaultValue("") .width(ConfigDef.Width.LONG) .build() ).define( ConfigKeyBuilder.of(SCHEMA_GENERATION_ENABLED_CONF, Type.BOOLEAN) .documentation(SCHEMA_GENERATION_ENABLED_DOC) .importance(ConfigDef.Importance.MEDIUM) .group(GROUP_SCHEMA_GENERATION) .defaultValue(false) .recommender(schemaRecommender) .build() ) .define( ConfigKeyBuilder.of(SCHEMA_GENERATION_KEY_FIELDS_CONF, Type.LIST) .documentation(SCHEMA_GENERATION_KEY_FIELDS_DOC) .importance(ConfigDef.Importance.MEDIUM) .group(GROUP_SCHEMA_GENERATION) .defaultValue(ImmutableList.of()) .recommender(schemaRecommender) .build() ).define( ConfigKeyBuilder.of(SCHEMA_GENERATION_KEY_NAME_CONF, Type.STRING) .documentation(SCHEMA_GENERATION_KEY_NAME_DOC) .importance(ConfigDef.Importance.MEDIUM) .group(GROUP_SCHEMA_GENERATION) .defaultValue("com.github.jcustenborder.kafka.connect.model.Key") .recommender(schemaRecommender) .build() ).define( ConfigKeyBuilder.of(SCHEMA_GENERATION_VALUE_NAME_CONF, Type.STRING) .documentation(SCHEMA_GENERATION_VALUE_NAME_DOC) .importance(ConfigDef.Importance.MEDIUM) .group(GROUP_SCHEMA_GENERATION) .defaultValue("com.github.jcustenborder.kafka.connect.model.Value") .recommender(schemaRecommender) .build() ) .define( ConfigKeyBuilder.of(PARSER_TIMESTAMP_TIMEZONE_CONF, Type.STRING) .documentation(PARSER_TIMESTAMP_TIMEZONE_DOC) .importance(ConfigDef.Importance.LOW) .group(GROUP_TIMESTAMP) .defaultValue(PARSER_TIMESTAMP_TIMEZONE_DEFAULT) .build() ).define( ConfigKeyBuilder.of(PARSER_TIMESTAMP_DATE_FORMATS_CONF, Type.LIST) .documentation(PARSER_TIMESTAMP_DATE_FORMATS_DOC) .importance(ConfigDef.Importance.LOW) .group(GROUP_TIMESTAMP) .defaultValue(PARSER_TIMESTAMP_DATE_FORMATS_DEFAULT) .build() ).define( ConfigKeyBuilder.of(TIMESTAMP_FIELD_CONF, Type.STRING) .documentation(TIMESTAMP_FIELD_DOC) .importance(ConfigDef.Importance.MEDIUM) .group(GROUP_TIMESTAMP) .defaultValue("") .recommender( Recommenders.visibleIf(TIMESTAMP_MODE_CONF, TimestampMode.FIELD.toString()) ) .build() ); } public abstract boolean schemasRequired(); Schema readSchema(final String key) { String schema = this.getString(key); Schema result; if (Strings.isNullOrEmpty(schema)) { result = null; } else { try { result = ObjectMapperFactory.INSTANCE.readValue(schema, Schema.class); } catch (IOException e) { throw new DataException("Could not read schema from '" + key + "'", e); } } return result; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSpoolDirSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.data.Parser; import com.github.jcustenborder.kafka.connect.utils.data.type.DateTypeParser; import com.github.jcustenborder.kafka.connect.utils.data.type.TimeTypeParser; import com.github.jcustenborder.kafka.connect.utils.data.type.TimestampTypeParser; import com.github.jcustenborder.kafka.connect.utils.data.type.TypeParser; import com.google.common.collect.ImmutableMap; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.data.Time; import org.apache.kafka.connect.data.Timestamp; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; import java.util.Map; public abstract class AbstractSpoolDirSourceTask extends AbstractSourceTask { private static final Logger log = LoggerFactory.getLogger(AbstractSpoolDirSourceTask.class); protected Parser parser; @Override public void start(Map settings) { super.start(settings); this.parser = new Parser(); Map dateTypeParsers = ImmutableMap.of( Timestamp.SCHEMA, new TimestampTypeParser(this.config.parserTimestampTimezone, this.config.parserTimestampDateFormats), Date.SCHEMA, new DateTypeParser(this.config.parserTimestampTimezone, this.config.parserTimestampDateFormats), Time.SCHEMA, new TimeTypeParser(this.config.parserTimestampTimezone, this.config.parserTimestampDateFormats) ); for (Map.Entry kvp : dateTypeParsers.entrySet()) { this.parser.registerTypeParser(kvp.getKey(), kvp.getValue()); } } protected void addRecord(List records, SchemaAndValue key, SchemaAndValue value) { final Long timestamp; switch (this.config.timestampMode) { case FIELD: Struct valueStruct = (Struct) value.value(); log.trace("addRecord() - Reading date from timestamp field '{}'", this.config.timestampField); final java.util.Date date = (java.util.Date) valueStruct.get(this.config.timestampField); timestamp = date.getTime(); break; case FILE_TIME: timestamp = this.inputFile.lastModified(); break; case PROCESS_TIME: timestamp = null; break; default: throw new UnsupportedOperationException( String.format("Unsupported timestamp mode. %s", this.config.timestampMode) ); } SourceRecord sourceRecord = record( key, value, timestamp ); recordCount++; records.add(sourceRecord); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractTaskPartitionerPredicate.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.hash.Hashing; import org.apache.kafka.common.config.ConfigException; import java.io.File; import java.util.function.Predicate; abstract class AbstractTaskPartitionerPredicate implements Predicate { final int index; final int count; protected AbstractTaskPartitionerPredicate(int index, int count) { this.index = index; this.count = count; } public static Predicate create(AbstractSourceConnectorConfig config) { Predicate result; if (config.taskCount == 1) { result = new None(config.taskIndex, config.taskCount); } else { switch (config.taskPartitioner) { case ByName: result = new ByName(config.taskIndex, config.taskCount); break; default: throw new ConfigException( AbstractSourceConnectorConfig.TASK_PARTITIONER_CONF, config.taskPartitioner.toString(), "Unsupported value" ); } } return result; } /** * This implementation is used to bypass the check. */ static class None extends AbstractTaskPartitionerPredicate { None(int index, int count) { super(index, count); } @Override public boolean test(File file) { return true; } } /** * */ static class ByName extends AbstractTaskPartitionerPredicate { protected ByName(int index, int count) { super(index, count); } @Override public boolean test(File file) { final int hashcode = Math.abs( Hashing.adler32() .hashUnencodedChars(file.getName()) .asInt() ); return (hashcode % this.count) == index; } } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/CsvSchemaGenerator.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; import com.opencsv.ICSVParser; import org.apache.kafka.connect.data.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.LinkedHashMap; import java.util.Map; public class CsvSchemaGenerator extends AbstractSchemaGenerator { private static final Logger log = LoggerFactory.getLogger(CsvSchemaGenerator.class); public CsvSchemaGenerator(Map settings) { super(settings); } @Override protected SpoolDirCsvSourceConnectorConfig config(Map settings) { return new SpoolDirCsvSourceConnectorConfig(false, settings); } @Override protected Map determineFieldTypes(InputStream inputStream) throws IOException { Map typeMap = new LinkedHashMap<>(); ICSVParser parserBuilder = this.config.createCSVParserBuilder(); try (InputStreamReader reader = new InputStreamReader(inputStream)) { CSVReaderBuilder readerBuilder = this.config.createCSVReaderBuilder(reader, parserBuilder); try (CSVReader csvReader = readerBuilder.build()) { String[] headers = null; if (this.config.firstRowAsHeader) { headers = csvReader.readNext(); } String[] row = csvReader.readNext(); if (null == headers) { headers = new String[row.length]; for (int i = 1; i <= row.length; i++) { headers[(i - 1)] = String.format("column%02d", i); } } for (String s : headers) { typeMap.put(s, Schema.Type.STRING); } } } return typeMap; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/FileComparator.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.collect.ComparisonChain; import java.io.File; import java.util.Comparator; import java.util.List; class FileComparator implements Comparator { final List attributes; FileComparator(List attributes) { this.attributes = attributes; } @Override public int compare(File f1, File f2) { ComparisonChain chain = ComparisonChain.start(); for (AbstractSourceConnectorConfig.FileAttribute fileAttribute : this.attributes) { switch (fileAttribute) { case NameAsc: chain = chain.compare(f1.getName(), f2.getName()); break; case NameDesc: chain = chain.compare(f2.getName(), f1.getName()); break; case LengthAsc: // We prefer larger files first. chain = chain.compare(f1.length(), f2.length()); break; case LengthDesc: // We prefer larger files first. chain = chain.compare(f2.length(), f1.length()); break; case LastModifiedAsc: chain = chain.compare(f1.lastModified(), f2.lastModified()); break; case LastModifiedDesc: chain = chain.compare(f2.lastModified(), f1.lastModified()); break; default: throw new UnsupportedOperationException( String.format("%s is not a supported FileAttribute.", fileAttribute) ); } } return chain.result(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/InputFile.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; import org.apache.commons.compress.compressors.CompressorException; import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.Closeable; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.nio.charset.Charset; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; public class InputFile implements Closeable { private static final Logger log = LoggerFactory.getLogger(InputFile.class); private final File file; private final File processingFlag; private final String name; private final String path; private final long length; private final long lastModified; private final Metadata metadata; private final AbstractSourceConnectorConfig config; private final String inputPathSubDir; InputStreamReader inputStreamReader; LineNumberReader lineNumberReader; InputStream inputStream; InputFile(AbstractSourceConnectorConfig config, File file) { this.config = config; this.file = file; this.name = this.file.getName(); this.path = this.file.getPath(); this.lastModified = this.file.lastModified(); this.length = this.file.length(); String processingFileName = file.getName() + config.processingFileExtension; this.processingFlag = new File(file.getParentFile(), processingFileName); this.inputPathSubDir = determineRelativePath(file, config.inputPath); this.metadata = new Metadata(file, this.inputPathSubDir); } static final Map SUPPORTED_COMPRESSION_TYPES = ImmutableMap.of( "bz2", CompressorStreamFactory.BZIP2, "gz", CompressorStreamFactory.GZIP, "snappy", CompressorStreamFactory.SNAPPY_RAW, "lz4", CompressorStreamFactory.LZ4_BLOCK, "z", CompressorStreamFactory.Z ); private static String determineRelativePath(File inputPath, File inputFile) { Path relative = inputFile.toPath().relativize(inputPath.toPath()); // inputPath.toPath().relativize(inputFile.getParentFile().toPath()); String subDir = relative.toString(); if ("".equals(subDir)) { return null; } return subDir; } public String inputPathSubDir() { return this.inputPathSubDir; } public File file() { return this.file; } public File processingFlag() { return this.processingFlag; } public Metadata metadata() { return this.metadata; } public InputStream inputStream() { return this.inputStream; } public InputStream openStream() throws IOException { if (null != this.inputStream) { throw new IOException( String.format("File %s is already open", this.file) ); } final String extension = Files.getFileExtension(file.getName()); log.trace("openStream() - fileName = '{}' extension = '{}'", file, extension); this.inputStream = new FileInputStream(this.file); if (this.config.bufferedInputStream) { log.trace( "openStream() - Wrapping '{}' in a BufferedInputStream with bufferSize = {}", this.file, this.config.fileBufferSizeBytes ); this.inputStream = new BufferedInputStream(this.inputStream, this.config.fileBufferSizeBytes); } if (SUPPORTED_COMPRESSION_TYPES.containsKey(extension)) { final String compressor = SUPPORTED_COMPRESSION_TYPES.get(extension); log.info("Decompressing {} as {}", file, compressor); final CompressorStreamFactory compressorStreamFactory = new CompressorStreamFactory(); try { this.inputStream = compressorStreamFactory.createCompressorInputStream( compressor, this.inputStream ); } catch (CompressorException e) { throw new IOException("Exception thrown while creating compressor stream " + compressor, e); } } startProcessing(); return inputStream; } public void startProcessing() throws IOException { log.info("Creating processing flag {}", this.processingFlag); Files.touch(this.processingFlag); } public InputStreamReader openInputStreamReader(Charset charset) throws IOException { if (null == this.inputStreamReader) { InputStream inputStream = null != this.inputStream ? this.inputStream : openStream(); this.inputStreamReader = new InputStreamReader(inputStream, charset); } return this.inputStreamReader; } public InputStreamReader inputStreamReader() { return this.inputStreamReader; } public LineNumberReader openLineNumberReader(Charset charset) throws IOException { if (null == this.lineNumberReader) { InputStreamReader inputStreamReader = this.inputStreamReader != null ? this.inputStreamReader : openInputStreamReader(charset); this.lineNumberReader = new LineNumberReader(inputStreamReader); } return this.lineNumberReader; } public LineNumberReader lineNumberReader() { return this.lineNumberReader; } @Override public String toString() { return this.file.toString(); } @Override public void close() throws IOException { if (null != this.lineNumberReader) { this.lineNumberReader.close(); } if (null != this.inputStreamReader) { this.inputStreamReader.close(); } if (null != this.inputStream) { log.info("Closing {}", this.file); this.inputStream.close(); } if (this.processingFlag.exists()) { log.info("Removing processing flag {}", this.processingFlag); if (!this.processingFlag.delete()) { log.warn("Could not remove processing flag {}", this.processingFlag); } } } public String getName() { return name; } public String getPath() { return path; } public long length() { return this.length; } public long lastModified() { return this.lastModified; } private List getInputPathSubDirsToCleanup() { List inputPathSubDirsToCleanup = null; if (this.inputPathSubDir != null && !config.inputPathWalkRecursivelyRetainSubDirs) { inputPathSubDirsToCleanup = new ArrayList(); File lastSubDir = this.config.inputPath; for (String subDirName : this.inputPathSubDir.split(File.separator)) { lastSubDir = new File(lastSubDir, subDirName); inputPathSubDirsToCleanup.add(lastSubDir); } Collections.reverse(inputPathSubDirsToCleanup); } return inputPathSubDirsToCleanup; } private void cleanupInputDirSubDirs() { List inputPathSubDirsToCleanup = this.getInputPathSubDirsToCleanup(); if (inputPathSubDirsToCleanup != null) { for (File subDir : inputPathSubDirsToCleanup) { try { if (subDir.listFiles() == null || subDir.listFiles().length == 0) { if (!subDir.delete()) { log.error("Failed to delete input.path sub-directory: {}", subDir); } else { log.info("Cleaned up input.path sub-directory: {}", subDir); } } else { log.info("Cannot clean up input.path sub-directory as it is not empty: {}", subDir); } } catch (SecurityException e) { log.error("SecurityException thrown while trying to delete input.path sub-directory: {}", subDir, e); } } } } public void moveToDirectory(File outputDirectory) { if (this.inputPathSubDir != null) { outputDirectory = new File(outputDirectory, this.inputPathSubDir); if (!outputDirectory.isDirectory()) { outputDirectory.mkdirs(); } } File outputFile = new File(outputDirectory, this.file.getName()); try { if (this.file.exists()) { log.info("Moving {} to {}", this.file, outputFile); Files.move(this.file, outputFile); } } catch (IOException e) { log.error("Exception thrown while trying to move {} to {}", this.file, outputFile, e); } this.cleanupInputDirSubDirs(); } public void delete() { log.info("Deleting {}", this.file); if (!this.file.delete()) { log.warn("Could not delete {}", this.file); } this.cleanupInputDirSubDirs(); } public boolean exists() { return this.file.exists(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/InputFileDequeue.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.collect.ForwardingDeque; import com.google.common.io.PatternFilenameFilter; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayDeque; import java.util.Arrays; import java.util.Deque; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; public class InputFileDequeue extends ForwardingDeque { private static final Logger log = LoggerFactory.getLogger(InputFileDequeue.class); private final AbstractSourceConnectorConfig config; private final FileComparator fileComparator; private final Predicate processingFileExists; private final Predicate fileMinimumAge; private final Predicate filePartitionSelector; public InputFileDequeue(AbstractSourceConnectorConfig config) { this.config = config; this.fileComparator = new FileComparator(config.fileSortAttributes); this.processingFileExists = new ProcessingFileExistsPredicate(config.processingFileExtension); this.fileMinimumAge = new MinimumFileAgePredicate(config.minimumFileAgeMS); this.filePartitionSelector = AbstractTaskPartitionerPredicate.create(config); } Deque files; static File processingFile(String processingFileExtension, File input) { String fileName = input.getName() + processingFileExtension; return new File(input.getParentFile(), fileName); } @Override protected Deque delegate() { if (null != files && !files.isEmpty()) { return files; } log.trace("delegate() - Searching for file(s) in {}", this.config.inputPath); final File[] input; if (this.config.inputPathWalkRecursively) { final PatternFilenameFilter walkerFilenameFilter = this.config.inputFilenameFilter; Predicate filenameFilterPredicate = file -> walkerFilenameFilter.accept(file.getParentFile(), file.getName()); try (Stream filesWalk = Files.walk(this.config.inputPath.toPath())) { input = filesWalk.map(Path::toFile) .filter(File::isFile) .filter(filenameFilterPredicate) .toArray(File[]::new); } catch (IOException e) { log.error("Unexpected eror walking {}: {}", this.config.inputPath.toPath(), e.getMessage(), e); return new ArrayDeque<>(); } } else { input = this.config.inputPath.listFiles(this.config.inputFilenameFilter); } if (null == input || input.length == 0) { log.info("No files matching {} were found in {}", AbstractSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, this.config.inputPath); return new ArrayDeque<>(); } log.trace("delegate() - Found {} potential file(s).", input.length); this.files = Arrays.stream(input) .filter(this.filePartitionSelector) .filter(this.processingFileExists) .filter(this.fileMinimumAge) .sorted(this.fileComparator) .map(f -> new InputFile(this.config, f)) .collect(Collectors.toCollection(ArrayDeque::new)); return this.files; } static class ProcessingFileExistsPredicate implements Predicate { final String processingFileExtension; ProcessingFileExistsPredicate(String processingFileExtension) { this.processingFileExtension = processingFileExtension; } @Override public boolean test(File file) { File processingFile = processingFile(this.processingFileExtension, file); log.trace("Checking for processing file: {}", processingFile); return !processingFile.exists(); } } static class MinimumFileAgePredicate implements Predicate { final long minimumFileAgeMS; final Time time; /** * @param minimumFileAgeMS Minimum time since last write in milliseconds. */ MinimumFileAgePredicate(long minimumFileAgeMS) { this(minimumFileAgeMS, Time.SYSTEM); } /** * Constructor is only used for testing. * * @param minimumFileAgeMS * @param time */ MinimumFileAgePredicate(long minimumFileAgeMS, Time time) { this.minimumFileAgeMS = minimumFileAgeMS; this.time = time; } @Override public boolean test(File file) { long fileAgeMS = this.time.milliseconds() - file.lastModified(); if (fileAgeMS < 0L) { log.warn("File {} has a date in the future.", file); } if (fileAgeMS >= this.minimumFileAgeMS) { return true; } else { log.debug("Skipping {} because it does not meet the minimum age.", file); return false; } } } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/JsonSchemaGenerator.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import org.apache.kafka.connect.data.Schema; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; public class JsonSchemaGenerator extends AbstractSchemaGenerator { public JsonSchemaGenerator(Map settings) { super(settings); } @Override protected SpoolDirJsonSourceConnectorConfig config(Map settings) { return new SpoolDirJsonSourceConnectorConfig(false, settings); } @Override protected Map determineFieldTypes(InputStream inputStream) throws IOException { Map typeMap = new LinkedHashMap<>(); JsonFactory factory = new JsonFactory(); try (JsonParser parser = factory.createParser(inputStream)) { Iterator iterator = ObjectMapperFactory.INSTANCE.readValues(parser, JsonNode.class); while (iterator.hasNext()) { JsonNode node = iterator.next(); if (node.isObject()) { Iterator fieldNames = node.fieldNames(); while (fieldNames.hasNext()) { typeMap.put(fieldNames.next(), Schema.Type.STRING); } break; } } } return typeMap; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/Metadata.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; import org.apache.kafka.connect.header.ConnectHeaders; import org.apache.kafka.connect.header.Headers; import java.io.File; import java.util.Date; import java.util.LinkedHashMap; import java.util.Map; /** * Class is used to write metadata for the InputFile. */ class Metadata { static final String HEADER_PATH = "file.path"; static final String HEADER_PARENT_DIR_NAME = "file.parent.dir.name"; static final String HEADER_NAME = "file.name"; static final String HEADER_NAME_WITHOUT_EXTENSION = "file.name.without.extension"; static final String HEADER_LAST_MODIFIED = "file.last.modified"; static final String HEADER_LENGTH = "file.length"; static final String HEADER_OFFSET = "file.offset"; static final String HEADER_FILE_RELATIVE_PATH = "file.relative.path"; final String path; final String name; final String nameWithoutExtension; final Date lastModified; final long length; final String relativePath; String parentDirName = null; public static final Map HEADER_DESCRIPTIONS; static { Map result = new LinkedHashMap<>(); result.put(HEADER_PATH, "The absolute path to the file ingested."); result.put(HEADER_PARENT_DIR_NAME, "The parent directory name of the file ingested"); result.put(HEADER_NAME, "The name part of the file ingested."); result.put(HEADER_NAME_WITHOUT_EXTENSION, "The file name without the extension part of the file."); result.put(HEADER_LAST_MODIFIED, "The last modified date of the file."); result.put(HEADER_LENGTH, "The size of the file in bytes."); result.put(HEADER_OFFSET, "The offset for this piece of data within the file."); result.put(HEADER_FILE_RELATIVE_PATH, "The file's parent sub-directory relative from the input.path."); HEADER_DESCRIPTIONS = ImmutableMap.copyOf(result); } public static final String HEADER_DOCS; static { StringBuilder builder = new StringBuilder(); HEADER_DESCRIPTIONS.forEach((key, value) -> { builder.append("* `"); builder.append(key); builder.append("` - "); builder.append(value); builder.append('\n'); }); HEADER_DOCS = builder.toString(); } public Metadata(File file, String relativePath) { this.path = file.getAbsolutePath(); this.name = file.getName(); this.lastModified = new Date(file.lastModified()); this.length = file.length(); this.nameWithoutExtension = Files.getNameWithoutExtension(this.name); if (file.getParentFile() != null) { this.parentDirName = file.getParentFile().getName(); } this.relativePath = relativePath; } /** * Method is used to copy metadata from the file to the headers of the file. * * @return Returns a Headers object populated with the metadata from the file. */ public Headers headers(long offset) { ConnectHeaders headers = new ConnectHeaders(); headers.addString(HEADER_NAME, this.name); headers.addString(HEADER_NAME_WITHOUT_EXTENSION, this.nameWithoutExtension); headers.addString(HEADER_PATH, this.path); headers.addString(HEADER_PARENT_DIR_NAME, this.parentDirName); headers.addLong(HEADER_LENGTH, this.length); headers.addLong(HEADER_OFFSET, offset); headers.addTimestamp(HEADER_LAST_MODIFIED, this.lastModified); if (this.relativePath != null) { headers.addString(HEADER_FILE_RELATIVE_PATH, this.relativePath); } return headers; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirAvroSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.Description; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationImportant; import com.github.jcustenborder.kafka.connect.utils.config.Title; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; import java.util.Map; @Title("Avro Source Connector") @Description("This connector is used to read avro data files from the file system and write their contents " + "to Kafka. The schema of the file is used to read the data and produce it to Kafka") @DocumentationImportant("This connector has a dependency on the Confluent Schema Registry specifically kafka-connect-avro-converter. " + "This dependency is not shipped along with the connector to ensure that there are not potential version mismatch issues. " + "The easiest way to ensure this component is available is to use one of the Confluent packages or containers for deployment.") public class SpoolDirAvroSourceConnector extends AbstractSourceConnector { @Override protected SpoolDirAvroSourceConnectorConfig config(Map settings) { return new SpoolDirAvroSourceConnectorConfig(settings); } @Override public Class taskClass() { return SpoolDirAvroSourceTask.class; } @Override public ConfigDef config() { return SpoolDirAvroSourceConnectorConfig.config(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirAvroSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.apache.kafka.common.config.ConfigDef; import java.util.Map; public class SpoolDirAvroSourceConnectorConfig extends AbstractSourceConnectorConfig { public SpoolDirAvroSourceConnectorConfig(Map originals) { super(config(), originals, true); } public static ConfigDef config() { return AbstractSourceConnectorConfig.config(true); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirAvroSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import io.confluent.connect.avro.AvroData; import org.apache.avro.file.DataFileReader; import org.apache.avro.generic.GenericContainer; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.io.DatumReader; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; public class SpoolDirAvroSourceTask extends AbstractSourceTask { private static final Logger log = LoggerFactory.getLogger(SpoolDirAvroSourceTask.class); long recordOffset; AvroData avroData = new AvroData(1024); DataFileReader dataFileReader; DatumReader datumReader = new GenericDatumReader<>(); @Override protected SpoolDirAvroSourceConnectorConfig config(Map settings) { return new SpoolDirAvroSourceConnectorConfig(settings); } @Override protected void configure(InputFile inputFile, Long lastOffset) throws IOException { if (null != this.dataFileReader) { this.dataFileReader.close(); } inputFile.startProcessing(); this.dataFileReader = new DataFileReader<>(inputFile.file(), datumReader); this.recordOffset = 0; if (null != lastOffset) { while (recordOffset < lastOffset && this.dataFileReader.hasNext()) { this.dataFileReader.next(); recordOffset++; } } } @Override protected List process() throws IOException { int recordCount = 0; List records = new ArrayList<>(this.config.batchSize); GenericContainer container = null; while (recordCount <= this.config.batchSize && dataFileReader.hasNext()) { container = dataFileReader.next(container); SchemaAndValue value = avroData.toConnectData(this.dataFileReader.getSchema(), container); SourceRecord sourceRecord = record(null, value, null); records.add(sourceRecord); recordCount++; recordOffset++; } return records; } @Override protected long recordOffset() { return recordOffset; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirBinaryFileSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.Description; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationImportant; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationWarning; import com.github.jcustenborder.kafka.connect.utils.config.Title; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; import java.util.Map; @Title("Binary File Source Connector") @Description("This connector is used to read an entire file as a byte array write the data to Kafka.") @DocumentationImportant("The recommended converter to use is the ByteArrayConverter. " + "Example: `value.converter=org.apache.kafka.connect.storage.ByteArrayConverter`") @DocumentationWarning("Large files will be read as a single byte array. This means that the process could " + "run out of memory or try to send a message to Kafka that is greater than the max message size. If this happens " + "an exception will be thrown.") public class SpoolDirBinaryFileSourceConnector extends AbstractSourceConnector { @Override protected SpoolDirBinaryFileSourceConnectorConfig config(Map settings) { return new SpoolDirBinaryFileSourceConnectorConfig(settings); } @Override public Class taskClass() { return SpoolDirBinaryFileSourceTask.class; } @Override public ConfigDef config() { return SpoolDirBinaryFileSourceConnectorConfig.config(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirBinaryFileSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.apache.kafka.common.config.ConfigDef; import java.util.Map; public class SpoolDirBinaryFileSourceConnectorConfig extends AbstractSourceConnectorConfig { public SpoolDirBinaryFileSourceConnectorConfig(Map originals) { super(config(), originals, true); } public static ConfigDef config() { return AbstractSourceConnectorConfig.config(true); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirBinaryFileSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.ByteStreams; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; public class SpoolDirBinaryFileSourceTask extends AbstractSourceTask { private static final Logger log = LoggerFactory.getLogger(SpoolDirBinaryFileSourceTask.class); @Override protected SpoolDirBinaryFileSourceConnectorConfig config(Map settings) { return new SpoolDirBinaryFileSourceConnectorConfig(settings); } @Override protected void configure(InputFile inputFile, Long lastOffset) throws IOException { inputFile.openStream(); } @Override protected List process() throws IOException { List records = new ArrayList<>(1); if (this.inputFile.inputStream().available() > 0) { byte[] buffer = ByteStreams.toByteArray(this.inputFile.inputStream()); records.add( record( null, new SchemaAndValue(Schema.BYTES_SCHEMA, buffer), null ) ); } return records; } @Override protected long recordOffset() { return 0; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.Description; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationImportant; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationTip; import com.github.jcustenborder.kafka.connect.utils.config.Title; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; import java.util.Map; @Title("CSV Source Connector") @Description("The SpoolDirCsvSourceConnector will monitor the directory specified in `input.path` for files and read them as a CSV " + "converting each of the records to the strongly typed equivalent specified in `key.schema` and `value.schema`.") @DocumentationTip("To get a starting point for a schema you can use the following command to generate an all String schema. This " + "will give you the basic structure of a schema. From there you can changes the types to match what you expect.\n\n" + ".. code-block:: bash\n\n" + " mvn clean package\n" + " export CLASSPATH=\"$(find target/kafka-connect-target/usr/share/kafka-connect/kafka-connect-spooldir -type f -name '*.jar' | tr '\\n' ':')\"\n" + " kafka-run-class com.github.jcustenborder.kafka.connect.spooldir.AbstractSchemaGenerator -t csv -f src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.data -c config/CSVExample.properties -i id\n" + "") @DocumentationImportant("There are some caveats to running this connector with `" + SpoolDirCsvSourceConnectorConfig.SCHEMA_GENERATION_ENABLED_CONF + " = true`. If schema generation is enabled the connector will start by reading one of the files that match `" + SpoolDirCsvSourceConnectorConfig.INPUT_FILE_PATTERN_CONF + "` in the path specified by `" + SpoolDirCsvSourceConnectorConfig.INPUT_PATH_CONFIG + "`. If there are no files when the connector starts or is restarted " + "the connector will fail to start. If there are different fields in other files they will not be detected. The recommended path is to specify a schema that the " + "files will be parsed with. This will ensure that data written by this connector to Kafka will be consistent across files that have inconsistent columns. For example " + "if some files have an optional column that is not always included, create a schema that includes the column marked as optional.") public class SpoolDirCsvSourceConnector extends AbstractSpoolDirSourceConnector { @Override protected SpoolDirCsvSourceConnectorConfig config(Map settings) { return new SpoolDirCsvSourceConnectorConfig(false, settings); } @Override protected AbstractSchemaGenerator generator(Map settings) { return new CsvSchemaGenerator(settings); } @Override public Class taskClass() { return SpoolDirCsvSourceTask.class; } @Override public ConfigDef config() { return SpoolDirCsvSourceConnectorConfig.config(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils; import com.github.jcustenborder.kafka.connect.utils.config.ValidEnum; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.opencsv.CSVParser; import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; import com.opencsv.ICSVParser; import com.opencsv.RFC4180ParserBuilder; import com.opencsv.enums.CSVReaderNullFieldIndicator; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.errors.DataException; import java.io.Reader; import java.nio.charset.Charset; import java.util.Map; class SpoolDirCsvSourceConnectorConfig extends AbstractSpoolDirSourceConnectorConfig { //CSVRecordProcessorConfig public static final String CSV_SKIP_LINES_CONF = "csv.skip.lines"; public static final String CSV_SEPARATOR_CHAR_CONF = "csv.separator.char"; public static final String CSV_QUOTE_CHAR_CONF = "csv.quote.char"; public static final String CSV_ESCAPE_CHAR_CONF = "csv.escape.char"; public static final String CSV_STRICT_QUOTES_CONF = "csv.strict.quotes"; public static final String CSV_IGNORE_LEADING_WHITESPACE_CONF = "csv.ignore.leading.whitespace"; public static final String CSV_IGNORE_QUOTATIONS_CONF = "csv.ignore.quotations"; public static final String CSV_KEEP_CARRIAGE_RETURN_CONF = "csv.keep.carriage.return"; public static final String CSV_VERIFY_READER_CONF = "csv.verify.reader"; public static final String CSV_NULL_FIELD_INDICATOR_CONF = "csv.null.field.indicator"; public static final String CSV_FIRST_ROW_AS_HEADER_CONF = "csv.first.row.as.header"; public static final String CSV_CHARSET_CONF = "csv.file.charset"; public static final String CSV_CASE_SENSITIVE_FIELD_NAMES_CONF = "csv.case.sensitive.field.names"; public static final String CSV_USE_RFC_4180_PARSER_CONF = "csv.rfc.4180.parser.enabled"; static final String CSV_SKIP_LINES_DISPLAY = "Skip lins"; static final String CSV_SEPARATOR_CHAR_DISPLAY = "Separator Character"; static final String CSV_QUOTE_CHAR_DISPLAY = "Quote Character"; static final String CSV_ESCAPE_CHAR_DISPLAY = "Escape Character"; static final String CSV_STRICT_QUOTES_DISPLAY = "Strict Quotes"; static final String CSV_IGNORE_LEADING_WHITESPACE_DISPLAY = "Ignore leading whitespace"; static final String CSV_IGNORE_QUOTATIONS_DISPLAY = "Ignore quotations"; static final String CSV_KEEP_CARRIAGE_RETURN_DISPLAY = "Preserve Carriage Return?"; static final String CSV_VERIFY_READER_DISPLAY = "Verify reader"; static final String CSV_NULL_FIELD_INDICATOR_DISPLAY = "Null field indicator"; static final String CSV_FIRST_ROW_AS_HEADER_DISPLAY = "Treat first row as header."; static final String CSV_CHARSET_DISPLAY = "File character set."; static final String CSV_CASE_SENSITIVE_FIELD_NAMES_DISPLAY = "Case sensitive field names."; static final String CSV_USE_RFC_4180_PARSER_DISPLAY = "Flag to determine if the RFC 4180 should be " + "used instead."; static final Object CSV_USE_RFC_4180_PARSER_DEFAULT = false; static final String CSV_SKIP_LINES_DOC = "Number of lines to skip in the beginning of the file."; static final int CSV_SKIP_LINES_DEFAULT = CSVReader.DEFAULT_SKIP_LINES; static final String CSV_SEPARATOR_CHAR_DOC = "The character that separates each field in the form " + "of an integer. Typically in a CSV this is a ,(44) character. A TSV would use a tab(9) character. " + "If `" + CSV_SEPARATOR_CHAR_CONF + "` is defined as a null(0), then the RFC 4180 parser must be " + "utilized by default. This is the equivalent of `" + CSV_USE_RFC_4180_PARSER_CONF + " = true`."; static final int CSV_SEPARATOR_CHAR_DEFAULT = CSVParser.DEFAULT_SEPARATOR; static final int CSV_QUOTE_CHAR_DEFAULT = CSVParser.DEFAULT_QUOTE_CHARACTER; static final String CSV_ESCAPE_CHAR_DOC = "The character as an integer to use when a special " + "character is encountered. The default escape character is typically a \\(92)"; static final int CSV_ESCAPE_CHAR_DEFAULT = CSVParser.DEFAULT_ESCAPE_CHARACTER; static final String CSV_STRICT_QUOTES_DOC = "Sets the strict quotes setting - if true, characters outside the quotes are ignored."; static final boolean CSV_STRICT_QUOTES_DEFAULT = CSVParser.DEFAULT_STRICT_QUOTES; static final String CSV_IGNORE_LEADING_WHITESPACE_DOC = "Sets the ignore leading whitespace setting - if true, white space in front of a quote in a field is ignored."; static final boolean CSV_IGNORE_LEADING_WHITESPACE_DEFAULT = CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE; static final String CSV_IGNORE_QUOTATIONS_DOC = "Sets the ignore quotations mode - if true, quotations are ignored."; static final boolean CSV_IGNORE_QUOTATIONS_DEFAULT = CSVParser.DEFAULT_IGNORE_QUOTATIONS; static final String CSV_KEEP_CARRIAGE_RETURN_DOC = "Flag to determine if the carriage return at the end of the line should be maintained."; static final boolean CSV_KEEP_CARRIAGE_RETURN_DEFAULT = CSVReader.DEFAULT_KEEP_CR; static final String CSV_VERIFY_READER_DOC = "Flag to determine if the reader should be verified."; static final boolean CSV_VERIFY_READER_DEFAULT = CSVReader.DEFAULT_VERIFY_READER; static final String CSV_NULL_FIELD_INDICATOR_DOC = "Indicator to determine how the CSV Reader can determine if a field is null. Valid values are " + ConfigUtils.enumValues(CSVReaderNullFieldIndicator.class) + ". For more information see http://opencsv.sourceforge.net/apidocs/com/opencsv/enums/CSVReaderNullFieldIndicator.html."; static final String CSV_NULL_FIELD_INDICATOR_DEFAULT = CSVParser.DEFAULT_NULL_FIELD_INDICATOR.name(); static final String CSV_FIRST_ROW_AS_HEADER_DOC = "Flag to indicate if the fist row of data contains the header of the file. " + "If true the position of the columns will be determined by the first row to the CSV. The column position will be inferred " + "from the position of the schema supplied in `" + VALUE_SCHEMA_CONF + "`. If set to true the number of columns must be " + "greater than or equal to the number of fields in the schema."; static final boolean CSV_FIRST_ROW_AS_HEADER_DEFAULT = false; static final String CSV_CHARSET_DOC = "Character set to read wth file with."; static final String CSV_CHARSET_DEFAULT = Charset.defaultCharset().name(); static final String CSV_CASE_SENSITIVE_FIELD_NAMES_DOC = "Flag to determine if the field names in the header row should be treated as case sensitive."; static final String CSV_USE_RFC_4180_PARSER_DOC = "Flag to determine if the RFC 4180 parser should be used instead of the default parser."; static final String CSV_GROUP = "CSV Parsing"; private static final String CSV_QUOTE_CHAR_DOC = "The character that is used to quote a field. This typically happens when the " + CSV_SEPARATOR_CHAR_CONF + " character is within the data."; private static final Character NULL_CHAR = (char) 0; public final int skipLines; public final char separatorChar; public final char quoteChar; public final char escapeChar; public final boolean ignoreLeadingWhitespace; public final boolean ignoreQuotations; public final boolean strictQuotes; public final boolean keepCarriageReturn; public final boolean verifyReader; public final CSVReaderNullFieldIndicator nullFieldIndicator; public final boolean firstRowAsHeader; public final Charset charset; public final boolean caseSensitiveFieldNames; public final boolean useRFC4180Parser; public SpoolDirCsvSourceConnectorConfig(final boolean isTask, Map settings) { super(isTask, true, config(), settings); this.skipLines = this.getInt(SpoolDirCsvSourceConnectorConfig.CSV_SKIP_LINES_CONF); this.separatorChar = this.getChar(SpoolDirCsvSourceConnectorConfig.CSV_SEPARATOR_CHAR_CONF); this.quoteChar = this.getChar(SpoolDirCsvSourceConnectorConfig.CSV_QUOTE_CHAR_CONF); this.escapeChar = this.getChar(SpoolDirCsvSourceConnectorConfig.CSV_ESCAPE_CHAR_CONF); this.ignoreLeadingWhitespace = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_IGNORE_LEADING_WHITESPACE_CONF); this.ignoreQuotations = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_IGNORE_QUOTATIONS_CONF); this.strictQuotes = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_STRICT_QUOTES_CONF); this.keepCarriageReturn = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_KEEP_CARRIAGE_RETURN_CONF); this.verifyReader = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_VERIFY_READER_CONF); this.nullFieldIndicator = ConfigUtils.getEnum(CSVReaderNullFieldIndicator.class, this, SpoolDirCsvSourceConnectorConfig.CSV_NULL_FIELD_INDICATOR_CONF); this.firstRowAsHeader = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF); this.charset = ConfigUtils.charset(this, SpoolDirCsvSourceConnectorConfig.CSV_CHARSET_CONF); this.caseSensitiveFieldNames = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_CASE_SENSITIVE_FIELD_NAMES_CONF); this.useRFC4180Parser = this.getBoolean(CSV_USE_RFC_4180_PARSER_CONF); } static ConfigDef config() { return AbstractSpoolDirSourceConnectorConfig.config(true) .define( ConfigKeyBuilder.of(CSV_SKIP_LINES_CONF, ConfigDef.Type.INT) .defaultValue(CSV_SKIP_LINES_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_SKIP_LINES_DOC) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .displayName(CSV_SKIP_LINES_DISPLAY) .build() ) .define( ConfigKeyBuilder.of(CSV_SEPARATOR_CHAR_CONF, ConfigDef.Type.INT) .defaultValue(CSV_SEPARATOR_CHAR_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_SEPARATOR_CHAR_DOC) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .displayName(CSV_SEPARATOR_CHAR_DISPLAY) .build() ) .define( ConfigKeyBuilder.of(CSV_QUOTE_CHAR_CONF, ConfigDef.Type.INT) .defaultValue(CSV_QUOTE_CHAR_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_QUOTE_CHAR_DOC) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .displayName(CSV_QUOTE_CHAR_DISPLAY) .build() ) .define( ConfigKeyBuilder.of(CSV_ESCAPE_CHAR_CONF, ConfigDef.Type.INT) .defaultValue(CSV_ESCAPE_CHAR_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_ESCAPE_CHAR_DOC) .group(CSV_GROUP) .displayName(CSV_ESCAPE_CHAR_DISPLAY) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_STRICT_QUOTES_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(CSV_STRICT_QUOTES_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_STRICT_QUOTES_DOC) .displayName(CSV_STRICT_QUOTES_DISPLAY) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_IGNORE_LEADING_WHITESPACE_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(CSV_IGNORE_LEADING_WHITESPACE_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_IGNORE_LEADING_WHITESPACE_DOC) .group(CSV_GROUP) .displayName(CSV_IGNORE_LEADING_WHITESPACE_DISPLAY) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_IGNORE_QUOTATIONS_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(CSV_IGNORE_QUOTATIONS_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_IGNORE_QUOTATIONS_DOC) .displayName(CSV_IGNORE_QUOTATIONS_DISPLAY) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_KEEP_CARRIAGE_RETURN_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(CSV_KEEP_CARRIAGE_RETURN_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_KEEP_CARRIAGE_RETURN_DOC) .displayName(CSV_KEEP_CARRIAGE_RETURN_DISPLAY) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_VERIFY_READER_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(CSV_VERIFY_READER_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_VERIFY_READER_DOC) .group(CSV_GROUP) .displayName(CSV_VERIFY_READER_DISPLAY) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_NULL_FIELD_INDICATOR_CONF, ConfigDef.Type.STRING) .defaultValue(CSV_NULL_FIELD_INDICATOR_DEFAULT) .validator(ValidEnum.of(CSVReaderNullFieldIndicator.class)) .importance(ConfigDef.Importance.LOW) .documentation(CSV_NULL_FIELD_INDICATOR_DOC) .displayName(CSV_NULL_FIELD_INDICATOR_DISPLAY) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_FIRST_ROW_AS_HEADER_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(CSV_FIRST_ROW_AS_HEADER_DEFAULT) .importance(ConfigDef.Importance.MEDIUM) .documentation(CSV_FIRST_ROW_AS_HEADER_DOC) .displayName(CSV_FIRST_ROW_AS_HEADER_DISPLAY) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_CHARSET_CONF, ConfigDef.Type.STRING) .defaultValue(CSV_CHARSET_DEFAULT) .validator(CharsetValidator.of()) .importance(ConfigDef.Importance.LOW) .documentation(CSV_CHARSET_DOC) .displayName(CSV_CHARSET_DISPLAY) .group(CSV_GROUP) .width(ConfigDef.Width.LONG) .build() ) .define( ConfigKeyBuilder.of(CSV_CASE_SENSITIVE_FIELD_NAMES_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(false) .importance(ConfigDef.Importance.LOW) .documentation(CSV_CASE_SENSITIVE_FIELD_NAMES_DOC) .displayName(CSV_CASE_SENSITIVE_FIELD_NAMES_DISPLAY) .build() ) .define( ConfigKeyBuilder.of(CSV_USE_RFC_4180_PARSER_CONF, ConfigDef.Type.BOOLEAN) .defaultValue(CSV_USE_RFC_4180_PARSER_DEFAULT) .importance(ConfigDef.Importance.LOW) .documentation(CSV_USE_RFC_4180_PARSER_DOC) .displayName(CSV_USE_RFC_4180_PARSER_DISPLAY) .build() ); } final char getChar(String key) { int intValue = this.getInt(key); return (char) intValue; } public ICSVParser createCSVParserBuilder() { final ICSVParser result; if (NULL_CHAR.equals(this.separatorChar) || this.useRFC4180Parser) { result = new RFC4180ParserBuilder() .withQuoteChar(this.quoteChar) .withSeparator(this.separatorChar) .withFieldAsNull(this.nullFieldIndicator) .build(); } else { result = new CSVParserBuilder() .withEscapeChar(this.escapeChar) .withIgnoreLeadingWhiteSpace(this.ignoreLeadingWhitespace) .withIgnoreQuotations(this.ignoreQuotations) .withQuoteChar(this.quoteChar) .withSeparator(this.separatorChar) .withStrictQuotes(this.strictQuotes) .withFieldAsNull(this.nullFieldIndicator) .build(); } return result; } public CSVReaderBuilder createCSVReaderBuilder(Reader reader, ICSVParser parser) { return new CSVReaderBuilder(reader) .withCSVParser(parser) .withKeepCarriageReturn(this.keepCarriageReturn) .withSkipLines(this.skipLines) .withVerifyReader(this.verifyReader) .withFieldAsNull(nullFieldIndicator); } @Override public boolean schemasRequired() { return true; } static class CharsetValidator implements ConfigDef.Validator { static CharsetValidator of() { return new CharsetValidator(); } @Override public void ensureValid(String s, Object o) { try { Preconditions.checkState(o instanceof String); String input = (String) o; Charset.forName(input); } catch (IllegalArgumentException e) { throw new DataException( String.format("Charset '%s' is invalid for %s", o, s), e ); } } @Override public String toString() { return Joiner.on(",").join(Charset.availableCharsets().keySet()); } } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.base.Joiner; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; import com.opencsv.ICSVParser; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.DataException; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; import java.util.Map; public class SpoolDirCsvSourceTask extends AbstractSpoolDirSourceTask { private static final Logger log = LoggerFactory.getLogger(SpoolDirCsvSourceTask.class); String[] fieldNames; private ICSVParser csvParser; private CSVReader csvReader; // private InputStreamReader streamReader; @Override protected SpoolDirCsvSourceConnectorConfig config(Map settings) { return new SpoolDirCsvSourceConnectorConfig(true, settings); } @Override protected void configure(InputFile inputFile, final Long lastOffset) throws IOException { log.trace("configure() - creating csvParser"); this.csvParser = this.config.createCSVParserBuilder(); InputStreamReader streamReader = inputFile.openInputStreamReader(this.config.charset); CSVReaderBuilder csvReaderBuilder = this.config.createCSVReaderBuilder(streamReader, csvParser); this.csvReader = csvReaderBuilder.build(); String[] fieldNames; if (this.config.firstRowAsHeader) { log.trace("configure() - Reading the header row."); fieldNames = this.csvReader.readNext(); log.info("configure() - field names from header row. fields = {}", Joiner.on(", ").join(fieldNames)); } else { log.trace("configure() - Using fields from schema {}", this.config.valueSchema.name()); fieldNames = new String[this.config.valueSchema.fields().size()]; int index = 0; for (Field field : this.config.valueSchema.fields()) { fieldNames[index++] = field.name(); } log.info("configure() - field names from schema order. fields = {}", Joiner.on(", ").join(fieldNames)); } if (null != lastOffset) { log.info("Found previous offset. Skipping {} line(s).", lastOffset.intValue()); String[] row = null; while (null != (row = this.csvReader.readNext()) && this.csvReader.getLinesRead() < lastOffset) { log.trace("skipped row"); } } this.fieldNames = fieldNames; } @Override public void start(Map settings) { super.start(settings); } @Override public long recordOffset() { final long result; if (null == this.csvReader) { result = -1L; } else { result = this.csvReader.getLinesRead() - this.config.skipLines - (this.config.firstRowAsHeader ? 1 : 0); } return result; } @Override public List process() throws IOException { List records = new ArrayList<>(this.config.batchSize); while (records.size() < this.config.batchSize) { String[] row = this.csvReader.readNext(); if (null == row) { break; } if (row.length == 1 && null == row[0]) { break; } log.trace("process() - Row on line {} has {} field(s)", recordOffset(), row.length); Struct keyStruct = new Struct(this.config.keySchema); Struct valueStruct = new Struct(this.config.valueSchema); for (int i = 0; i < this.fieldNames.length; i++) { String fieldName = this.fieldNames[i]; log.trace("process() - Processing field {}", fieldName); String input = row[i]; log.trace("process() - input = '{}'", input); Object fieldValue = null; try { Field field = this.config.valueSchema.field(fieldName); if (null != field) { fieldValue = this.parser.parseString(field.schema(), input); log.trace("process() - output = '{}'", fieldValue); valueStruct.put(field, fieldValue); } else { log.trace("process() - Field {} is not defined in the schema.", fieldName); } } catch (Exception ex) { String message = String.format("Exception thrown while parsing data for '%s'. linenumber=%s", fieldName, this.recordOffset()); throw new DataException(message, ex); } Field keyField = this.config.keySchema.field(fieldName); if (null != keyField) { log.trace("process() - Setting key field '{}' to '{}'", keyField.name(), fieldValue); keyStruct.put(keyField, fieldValue); } } if (log.isInfoEnabled() && this.csvReader.getLinesRead() % ((long) this.config.batchSize * 20) == 0) { log.info("Processed {} lines of {}", this.csvReader.getLinesRead(), this.inputFile); } addRecord( records, new SchemaAndValue(keyStruct.schema(), keyStruct), new SchemaAndValue(valueStruct.schema(), valueStruct) ); } return records; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.Description; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationImportant; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationNote; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationTip; import com.github.jcustenborder.kafka.connect.utils.config.Title; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; import java.util.Map; @Title("Json Source Connector") @Description("This connector is used to `stream ` JSON files from a directory " + "while converting the data based on the schema supplied in the configuration.") @DocumentationTip("To get a starting point for a schema you can use the following command to generate an all String schema. This " + "will give you the basic structure of a schema. From there you can changes the types to match what you expect.\n" + ".. code-block:: bash\n\n" + " mvn clean package\n" + " export CLASSPATH=\"$(find target/kafka-connect-target/usr/share/kafka-connect/kafka-connect-spooldir -type f -name '*.jar' | tr '\\n' ':')\"\n" + " kafka-run-class com.github.jcustenborder.kafka.connect.spooldir.AbstractSchemaGenerator -t json -f src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FieldsMatch.data -c config/JsonExample.properties -i id\n" + "") @DocumentationImportant("There are some caveats to running this connector with `" + SpoolDirCsvSourceConnectorConfig.SCHEMA_GENERATION_ENABLED_CONF + " = true`. If schema generation is enabled the connector will start by reading one of the files that match `" + SpoolDirCsvSourceConnectorConfig.INPUT_FILE_PATTERN_CONF + "` in the path specified by `" + SpoolDirCsvSourceConnectorConfig.INPUT_PATH_CONFIG + "`. If there are no files when the connector starts or is restarted " + "the connector will fail to start. If there are different fields in other files they will not be detected. The recommended path is to specify a schema that the " + "files will be parsed with. This will ensure that data written by this connector to Kafka will be consistent across files that have inconsistent columns. For example " + "if some files have an optional column that is not always included, create a schema that includes the column marked as optional.") @DocumentationNote("If you want to import JSON node by node in the file and do not care about schemas, do not use this connector with Schema Generation enabled. " + "Take a look at the Schema Less Json Source Connector.") public class SpoolDirJsonSourceConnector extends AbstractSpoolDirSourceConnector { @Override protected SpoolDirJsonSourceConnectorConfig config(Map settings) { return new SpoolDirJsonSourceConnectorConfig(false, settings); } @Override protected AbstractSchemaGenerator generator(Map settings) { return new JsonSchemaGenerator(settings); } @Override public Class taskClass() { return SpoolDirJsonSourceTask.class; } @Override public ConfigDef config() { return SpoolDirJsonSourceConnectorConfig.config(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.apache.kafka.common.config.ConfigDef; import java.util.Map; class SpoolDirJsonSourceConnectorConfig extends AbstractSpoolDirSourceConnectorConfig { public SpoolDirJsonSourceConnectorConfig(final boolean isTask, Map settings) { super(isTask, true, config(), settings); } @Override public boolean schemasRequired() { return true; } public static ConfigDef config() { return AbstractSpoolDirSourceConnectorConfig.config(true); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.DataException; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; public class SpoolDirJsonSourceTask extends AbstractSpoolDirSourceTask { private static final Logger log = LoggerFactory.getLogger(SpoolDirJsonSourceTask.class); JsonFactory jsonFactory; JsonParser jsonParser; Iterator iterator; long offset; @Override protected SpoolDirJsonSourceConnectorConfig config(Map settings) { return new SpoolDirJsonSourceConnectorConfig(true, settings); } @Override public void start(Map settings) { super.start(settings); this.jsonFactory = new JsonFactory(); } @Override protected void configure(InputFile inputFile, Long lastOffset) throws IOException { if (null != jsonParser) { log.trace("configure() - Closing existing json parser."); jsonParser.close(); } InputStream inputStream = inputFile.openStream(); this.jsonParser = this.jsonFactory.createParser(inputStream); this.iterator = ObjectMapperFactory.INSTANCE.readValues(this.jsonParser, JsonNode.class); this.offset = -1; if (null != lastOffset) { int skippedRecords = 1; while (this.iterator.hasNext() && skippedRecords <= lastOffset) { next(); skippedRecords++; } log.trace("configure() - Skipped {} record(s).", skippedRecords); log.info("configure() - Starting on offset {}", this.offset); } } JsonNode next() { this.offset++; return this.iterator.next(); } @Override protected List process() { List records = new ArrayList<>(this.config.batchSize); while (this.iterator.hasNext() && records.size() < this.config.batchSize) { JsonNode node = next(); Struct valueStruct = new Struct(this.config.valueSchema); Struct keyStruct = new Struct(this.config.keySchema); log.trace("process() - input = {}", node); for (Field field : this.config.valueSchema.fields()) { JsonNode fieldNode = node.get(field.name()); log.trace("process() - field: {} input = '{}'", field.name(), fieldNode); Object fieldValue; try { fieldValue = this.parser.parseJsonNode(field.schema(), fieldNode); log.trace("process() - field: {} output = '{}'", field.name(), fieldValue); valueStruct.put(field, fieldValue); Field keyField = this.config.keySchema.field(field.name()); if (null != keyField) { log.trace("process() - Setting key field '{}' to '{}'", keyField.name(), fieldValue); keyStruct.put(keyField, fieldValue); } } catch (Exception ex) { String message = String.format("Exception thrown while parsing data for '%s'. linenumber=%s", field.name(), this.recordOffset()); throw new DataException(message, ex); } } addRecord( records, new SchemaAndValue(keyStruct.schema(), keyStruct), new SchemaAndValue(valueStruct.schema(), valueStruct) ); } return records; } @Override protected long recordOffset() { return this.offset; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirLineDelimitedSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.Description; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationImportant; import com.github.jcustenborder.kafka.connect.utils.config.Title; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; import java.util.Map; @Title("Line Delimited Source Connector") @Description("This connector is used to read a file line by line and write the data to Kafka.") @DocumentationImportant("The recommended converter to use is the StringConverter. " + "Example: `value.converter=org.apache.kafka.connect.storage.StringConverter`") public class SpoolDirLineDelimitedSourceConnector extends AbstractSourceConnector { @Override protected SpoolDirLineDelimitedSourceConnectorConfig config(Map settings) { return new SpoolDirLineDelimitedSourceConnectorConfig(settings); } @Override public Class taskClass() { return SpoolDirLineDelimitedSourceTask.class; } @Override public ConfigDef config() { return SpoolDirLineDelimitedSourceConnectorConfig.config(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirLineDelimitedSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils; import org.apache.kafka.common.config.ConfigDef; import java.nio.charset.Charset; import java.util.Map; public class SpoolDirLineDelimitedSourceConnectorConfig extends AbstractSourceConnectorConfig { public static final String CHARSET_CONF = "file.charset"; static final String CHARSET_DOC = "Character set to read wth file with."; static final String CHARSET_DEFAULT = Charset.defaultCharset().name(); static final String CHARSET_DISPLAY = "File character set."; public final Charset charset; public SpoolDirLineDelimitedSourceConnectorConfig(Map originals) { super(config(), originals, true); this.charset = ConfigUtils.charset(this, CHARSET_CONF); } public static ConfigDef config() { return AbstractSourceConnectorConfig.config(true) .define( ConfigKeyBuilder.of(CHARSET_CONF, ConfigDef.Type.STRING) .defaultValue(CHARSET_DEFAULT) .validator(SpoolDirCsvSourceConnectorConfig.CharsetValidator.of()) .importance(ConfigDef.Importance.LOW) .documentation(CHARSET_DOC) .displayName(CHARSET_DISPLAY) .width(ConfigDef.Width.LONG) .build() ); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirLineDelimitedSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; public class SpoolDirLineDelimitedSourceTask extends AbstractSourceTask { private static final Logger log = LoggerFactory.getLogger(SpoolDirLineDelimitedSourceTask.class); @Override protected SpoolDirLineDelimitedSourceConnectorConfig config(Map settings) { return new SpoolDirLineDelimitedSourceConnectorConfig(settings); } @Override protected void configure(InputFile inputFile, Long lastOffset) throws IOException { this.inputFile.openLineNumberReader(this.config.charset); } @Override protected List process() throws IOException { int recordCount = 0; List records = new ArrayList<>(this.config.batchSize); String line = null; while (recordCount < this.config.batchSize && null != (line = this.inputFile.lineNumberReader().readLine())) { SourceRecord record = record( null, new SchemaAndValue(Schema.STRING_SCHEMA, line), null ); records.add(record); recordCount++; } return records; } @Override protected long recordOffset() { long result = -1L; if (null != this.inputFile && null != this.inputFile.lineNumberReader()) { result = this.inputFile.lineNumberReader().getLineNumber(); } return result; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirSchemaLessJsonSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.Description; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationImportant; import com.github.jcustenborder.kafka.connect.utils.config.Title; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; import java.util.Map; @Title("Schema Less Json Source Connector") @Description("This connector is used to `stream _` JSON files from a directory. " + "This connector will read each file node by node writing each node as a record in Kafka." + "For example if your data file contains several json objects the connector will read from { to } " + "for each object and write each object to Kafka.") @DocumentationImportant("This connector does not try to convert the json records to a schema. " + "The recommended converter to use is the StringConverter. " + "Example: `value.converter=org.apache.kafka.connect.storage.StringConverter`") public class SpoolDirSchemaLessJsonSourceConnector extends AbstractSourceConnector { @Override protected SpoolDirSchemaLessJsonSourceConnectorConfig config(Map settings) { return new SpoolDirSchemaLessJsonSourceConnectorConfig(settings); } @Override public Class taskClass() { return SpoolDirSchemaLessJsonSourceTask.class; } @Override public ConfigDef config() { return SpoolDirSchemaLessJsonSourceConnectorConfig.config(); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirSchemaLessJsonSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils; import org.apache.kafka.common.config.ConfigDef; import java.nio.charset.Charset; import java.util.Map; public class SpoolDirSchemaLessJsonSourceConnectorConfig extends AbstractSourceConnectorConfig { public static final String CHARSET_CONF = "file.charset"; static final String CHARSET_DOC = "Character set to read wth file with."; static final String CHARSET_DEFAULT = Charset.defaultCharset().name(); static final String CHARSET_DISPLAY = "File character set."; public final Charset charset; public SpoolDirSchemaLessJsonSourceConnectorConfig(Map originals) { super(config(), originals, true); this.charset = ConfigUtils.charset(this, CHARSET_CONF); } public static ConfigDef config() { return AbstractSourceConnectorConfig.config(true) .define( ConfigKeyBuilder.of(CHARSET_CONF, ConfigDef.Type.STRING) .defaultValue(CHARSET_DEFAULT) .validator(SpoolDirCsvSourceConnectorConfig.CharsetValidator.of()) .importance(ConfigDef.Importance.LOW) .documentation(CHARSET_DOC) .displayName(CHARSET_DISPLAY) .width(ConfigDef.Width.LONG) .build() ); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirSchemaLessJsonSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MappingIterator; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; public class SpoolDirSchemaLessJsonSourceTask extends AbstractSourceTask { private static final Logger log = LoggerFactory.getLogger(SpoolDirSchemaLessJsonSourceTask.class); @Override protected SpoolDirSchemaLessJsonSourceConnectorConfig config(Map settings) { return new SpoolDirSchemaLessJsonSourceConnectorConfig(settings); } JsonParser parser; MappingIterator nodeIterator; long recordOffset; @Override protected void configure(InputFile inputFile, Long lastOffset) throws IOException { if (null != this.parser) { this.parser.close(); } this.recordOffset = 0; InputStream inputStream = inputFile.openStream(); this.parser = ObjectMapperFactory.INSTANCE.getJsonFactory().createParser(inputStream); this.nodeIterator = ObjectMapperFactory.INSTANCE.readValues(this.parser, JsonNode.class); } @Override protected List process() throws IOException { int recordCount = 0; List records = new ArrayList<>(this.config.batchSize); while (recordCount < this.config.batchSize && this.nodeIterator.hasNext()) { JsonNode node = this.nodeIterator.next(); String value = ObjectMapperFactory.INSTANCE.writeValueAsString(node); SourceRecord record = record( null, new SchemaAndValue(Schema.STRING_SCHEMA, value), null ); records.add(record); recordCount++; recordOffset++; } return records; } @Override protected long recordOffset() { return this.recordOffset; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/SchemaConversion.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf; import com.github.jcustenborder.kafka.connect.spooldir.elf.converters.LogFieldConverter; import com.github.jcustenborder.parsers.elf.LogEntry; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.data.Struct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; public class SchemaConversion { private static final Logger log = LoggerFactory.getLogger(SchemaConversion.class); private final Schema valueSchema; private final List valueConverters; SchemaConversion(Schema valueSchema, List valueConverters) { this.valueSchema = valueSchema; this.valueConverters = valueConverters; } static SchemaAndValue convert(Schema schema, List converters, LogEntry entry) { final SchemaAndValue result; if (null == schema) { result = SchemaAndValue.NULL; } else { Struct struct = new Struct(schema); for (LogFieldConverter converter : converters) { converter.convert(entry, struct); } struct.validate(); result = new SchemaAndValue(schema, struct); } return result; } public SchemaAndValue convert(LogEntry entry) { final SchemaAndValue value = convert(this.valueSchema, this.valueConverters, entry); return value; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/SchemaConversionBuilder.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf; import com.github.jcustenborder.kafka.connect.spooldir.elf.converters.LogFieldConverter; import com.github.jcustenborder.kafka.connect.spooldir.elf.converters.LogFieldConverterFactory; import com.github.jcustenborder.parsers.elf.ElfParser; import com.google.common.base.Preconditions; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.LocalDate; import java.time.LocalTime; import java.util.ArrayList; import java.util.List; import java.util.Map; public class SchemaConversionBuilder { private static final Logger log = LoggerFactory.getLogger(SchemaConversionBuilder.class); final ElfParser parser; public SchemaConversionBuilder(ElfParser parser) { this.parser = parser; } static String normalizeFieldName(String fieldName) { Preconditions.checkNotNull(fieldName, "fieldname cannot be null."); final String result = fieldName.replace('(', '_') .replace(")", "") .replace('-', '_') .toLowerCase(); return result; } public SchemaConversion build() { log.trace("build() - Building SchemaConversion"); final SchemaBuilder valueBuilder = SchemaBuilder.struct(); valueBuilder.name("com.github.jcustenborder.kafka.connect.spooldir.LogEntry"); LogFieldConverterFactory factory = new LogFieldConverterFactory(); List valueConverters = new ArrayList<>(); for (Map.Entry> entry : this.parser.fieldTypes().entrySet()) { final String logFieldName = entry.getKey(); final Class logFieldClass = entry.getValue(); final String connectFieldName = normalizeFieldName(logFieldName); log.trace("build() - Mapping log field '{}' to schema field '{}'", logFieldName, connectFieldName); final LogFieldConverter converter = factory.create( valueBuilder, logFieldClass, logFieldName, connectFieldName ); valueConverters.add(converter); } if (LocalDate.class.equals(this.parser.fieldTypes().get("date")) && LocalTime.class.equals(this.parser.fieldTypes().get("time"))) { log.trace("build() - found date and time field. Creating datetime field."); final LogFieldConverter converter = factory.createDateTime( valueBuilder, "date", "time", "datetime" ); valueConverters.add(converter); } final Schema valueSchema = valueBuilder.build(); return new SchemaConversion(valueSchema, valueConverters); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/SpoolDirELFSourceConnector.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf; import com.github.jcustenborder.kafka.connect.spooldir.AbstractSourceConnectorConfig; import com.github.jcustenborder.kafka.connect.utils.VersionUtil; import com.github.jcustenborder.kafka.connect.utils.config.Description; import com.github.jcustenborder.kafka.connect.utils.config.Title; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.source.SourceConnector; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @Title("Extended Log File Format Source Connector") @Description("This connector is used to stream `Extended Log File Format ` " + "files from a directory while converting the data to a strongly typed schema.") public class SpoolDirELFSourceConnector extends SourceConnector { @Override public List> taskConfigs(int taskCount) { List> result = new ArrayList<>(); for (int i = 0; i < taskCount; i++) { Map taskConfig = new LinkedHashMap<>(this.settings); taskConfig.put(AbstractSourceConnectorConfig.TASK_INDEX_CONF, Integer.toString(i)); taskConfig.put(AbstractSourceConnectorConfig.TASK_COUNT_CONF, Integer.toString(taskCount)); result.add(taskConfig); } return result; } @Override public void stop() { } @Override public String version() { return VersionUtil.version(this.getClass()); } Map settings; @Override public void start(Map settings) { SpoolDirELFSourceConnectorConfig config = new SpoolDirELFSourceConnectorConfig(settings); this.settings = settings; } @Override public Class taskClass() { return SpoolDirELFSourceTask.class; } @Override public ConfigDef config() { return SpoolDirELFSourceConnectorConfig.config(true); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/SpoolDirELFSourceConnectorConfig.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf; import com.github.jcustenborder.kafka.connect.spooldir.AbstractSourceConnectorConfig; import org.apache.kafka.common.config.ConfigDef; import java.util.Map; class SpoolDirELFSourceConnectorConfig extends AbstractSourceConnectorConfig { public SpoolDirELFSourceConnectorConfig(Map settings) { super(config(true), settings, true); } public static ConfigDef config(boolean bufferedInputStream) { return AbstractSourceConnectorConfig.config(bufferedInputStream); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/SpoolDirELFSourceTask.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf; import com.github.jcustenborder.kafka.connect.spooldir.AbstractSourceTask; import com.github.jcustenborder.kafka.connect.spooldir.InputFile; import com.github.jcustenborder.parsers.elf.ElfParser; import com.github.jcustenborder.parsers.elf.ElfParserBuilder; import com.github.jcustenborder.parsers.elf.LogEntry; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; public class SpoolDirELFSourceTask extends AbstractSourceTask { private static final Logger log = LoggerFactory.getLogger(SpoolDirELFSourceTask.class); ElfParser parser; ElfParserBuilder parserBuilder; SchemaConversion conversion; long offset; @Override protected SpoolDirELFSourceConnectorConfig config(Map settings) { return new SpoolDirELFSourceConnectorConfig(settings); } @Override public void start(Map settings) { super.start(settings); this.parserBuilder = ElfParserBuilder.of(); } @Override protected void configure(InputFile inputFile, Long lastOffset) throws IOException { if (null != this.parser) { log.trace("configure() - Closing existing parser."); this.parser.close(); } InputStream inputStream = inputFile.openStream(); this.parser = this.parserBuilder.build(inputStream); SchemaConversionBuilder builder = new SchemaConversionBuilder(this.parser); this.conversion = builder.build(); this.offset = -1; if (null != lastOffset) { int skippedRecords = 1; while (null != next() && skippedRecords <= lastOffset) { skippedRecords++; } log.trace("configure() - Skipped {} record(s).", skippedRecords); log.info("configure() - Starting on offset {}", this.offset); } } LogEntry next() throws IOException { this.offset++; return this.parser.next(); } @Override protected List process() { int recordCount = 0; List records = new ArrayList<>(this.config.batchSize); LogEntry entry; try { while (null != (entry = next()) && recordCount < this.config.batchSize) { log.trace("process() - Processing LogEntry: {}", entry); SchemaAndValue value = conversion.convert(entry); SourceRecord record = record(SchemaAndValue.NULL, value, null); records.add(record); recordCount++; } } catch (IOException ex) { throw new ConnectException(ex); } return records; } @Override protected long recordOffset() { return this.offset; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/converters/LocalDateLogFieldConverter.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf.converters; import org.apache.kafka.connect.data.Field; import java.sql.Date; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; public class LocalDateLogFieldConverter extends LogFieldConverter { private static final ZoneId ZONE_ID = ZoneId.of("UTC"); @Override protected Object convert(Object input) { final LocalDate localDate = (LocalDate) input; final Instant instant = localDate.atStartOfDay(ZONE_ID).toInstant(); return Date.from(instant); } public LocalDateLogFieldConverter(String logFieldName, Field field) { super(logFieldName, field); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/converters/LocalTimeLogFieldConverter.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf.converters; import org.apache.kafka.connect.data.Field; import java.sql.Date; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneOffset; public class LocalTimeLogFieldConverter extends LogFieldConverter { private static final LocalDate EPOCH_DATE = LocalDate.ofEpochDay(0); @Override protected Object convert(Object input) { final LocalTime localTime = (LocalTime) input; final Instant instant = localTime.atDate(EPOCH_DATE).toInstant(ZoneOffset.UTC); return Date.from(instant); } public LocalTimeLogFieldConverter(String logFieldName, Field field) { super(logFieldName, field); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/converters/LogFieldConverter.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf.converters; import com.github.jcustenborder.parsers.elf.LogEntry; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Struct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public abstract class LogFieldConverter { private static final Logger log = LoggerFactory.getLogger(LogFieldConverter.class); protected final String logFieldName; protected final Field field; protected abstract Object convert(Object input); public LogFieldConverter(String logFieldName, Field field) { this.logFieldName = logFieldName; this.field = field; } public void convert(LogEntry logEntry, Struct struct) { final Object input = logEntry.fieldData().get(this.logFieldName); final Object output; if (null == input) { output = null; } else { output = convert(input); } log.trace("convert() - Setting {} to {}", field.name(), output); struct.put(this.field, output); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/converters/LogFieldConverterFactory.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf.converters; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Time; import org.apache.kafka.connect.data.Timestamp; import java.time.LocalDate; import java.time.LocalTime; public class LogFieldConverterFactory { static Schema schema(Class logClass, String logFieldName) { final SchemaBuilder builder; if (LocalDate.class.equals(logClass)) { builder = Date.builder(); } else if (LocalTime.class.equals(logClass)) { builder = Time.builder(); } else if (Integer.class.equals(logClass)) { builder = SchemaBuilder.int32(); } else if (Long.class.equals(logClass)) { builder = SchemaBuilder.int64(); } else if (String.class.equals(logClass)) { builder = SchemaBuilder.string(); } else { throw new UnsupportedOperationException( String.format("%s is not a supported type.", logClass.getName()) ); } builder.optional(); return builder.build(); } private static final String LOGFIELD_PARAM = "logField"; public LogFieldConverter create( SchemaBuilder builder, Class logClass, String logFieldName, String schemaFieldName) { final Schema fieldSchema; final Field field; final LogFieldConverter converter; if (LocalDate.class.equals(logClass)) { fieldSchema = Date.builder() .optional() .parameter(LOGFIELD_PARAM, logFieldName) .build(); builder.field(schemaFieldName, fieldSchema); field = builder.field(schemaFieldName); converter = new LocalDateLogFieldConverter(logFieldName, field); } else if (LocalTime.class.equals(logClass)) { fieldSchema = Time.builder() .optional() .parameter(LOGFIELD_PARAM, logFieldName) .build(); builder.field(schemaFieldName, fieldSchema); field = builder.field(schemaFieldName); converter = new LocalTimeLogFieldConverter(logFieldName, field); } else if (Integer.class.equals(logClass)) { fieldSchema = SchemaBuilder.int32() .optional() .parameter(LOGFIELD_PARAM, logFieldName) .build(); builder.field(schemaFieldName, fieldSchema); field = builder.field(schemaFieldName); converter = new PrimitiveLogFieldConverter(logFieldName, field); } else if (Long.class.equals(logClass)) { fieldSchema = SchemaBuilder.int64() .optional() .parameter(LOGFIELD_PARAM, logFieldName) .build(); builder.field(schemaFieldName, fieldSchema); field = builder.field(schemaFieldName); converter = new PrimitiveLogFieldConverter(logFieldName, field); } else if (String.class.equals(logClass)) { fieldSchema = SchemaBuilder.string() .optional() .parameter(LOGFIELD_PARAM, logFieldName) .build(); builder.field(schemaFieldName, fieldSchema); field = builder.field(schemaFieldName); converter = new PrimitiveLogFieldConverter(logFieldName, field); } else if (Double.class.equals(logClass)) { fieldSchema = SchemaBuilder.float64() .optional() .parameter(LOGFIELD_PARAM, logFieldName) .build(); builder.field(schemaFieldName, fieldSchema); field = builder.field(schemaFieldName); converter = new PrimitiveLogFieldConverter(logFieldName, field); } else { throw new UnsupportedOperationException( String.format("%s is not a supported type.", logClass.getName()) ); } return converter; } public LogFieldConverter createDateTime(SchemaBuilder builder, String logEntryDateField, String logEntryTimeField, String connectTimestampField) { final Schema fieldSchema = Timestamp.builder() .optional() .parameter(LOGFIELD_PARAM, String.format("%s,%s", logEntryDateField, logEntryTimeField)) .build(); builder.field(connectTimestampField, fieldSchema); final Field field = builder.field(connectTimestampField); final LogFieldConverter converter = new TimestampLogFieldConverter( field, logEntryTimeField, logEntryDateField ); return converter; } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/converters/PrimitiveLogFieldConverter.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf.converters; import org.apache.kafka.connect.data.Field; public class PrimitiveLogFieldConverter extends LogFieldConverter { @Override protected Object convert(Object input) { return input; } public PrimitiveLogFieldConverter(String logFieldName, Field field) { super(logFieldName, field); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/elf/converters/TimestampLogFieldConverter.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf.converters; import com.github.jcustenborder.parsers.elf.LogEntry; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Struct; import java.sql.Date; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneOffset; public class TimestampLogFieldConverter extends LogFieldConverter { private final String timeField; private final String dateField; public TimestampLogFieldConverter(Field field, String timeField, String dateField) { super(null, field); this.timeField = timeField; this.dateField = dateField; } @Override protected Object convert(Object input) { return null; } @Override public void convert(LogEntry logEntry, Struct struct) { final LocalDate date = (LocalDate) logEntry.fieldData().get(this.dateField); final LocalTime time = (LocalTime) logEntry.fieldData().get(this.timeField); final Object value; if (null == date || null == time) { value = null; } else { final Instant instant = time.atDate(date).toInstant(ZoneOffset.UTC); value = Date.from(instant); } struct.put(this.field, value); } } ================================================ FILE: src/main/java/com/github/jcustenborder/kafka/connect/spooldir/package-info.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ @Introduction( "This Kafka Connect connector provides the capability to watch a directory for files and " + "read the data as new files are written to the input directory. Each of the records in the " + "input file will be converted based on the user supplied schema. The connectors in this project " + "handle all different kinds of use cases like ingesting json, csv, tsv, avro, or binary files.") @Title("Spool Dir") @DocumentationWarning("Running these connectors with multiple tasks requires a shared volume across " + "all of the Kafka Connect workers. Kafka Connect does not have a mechanism for synchronization of " + "tasks. Because of this each task will select which file it will use the following " + "algorithm `hash() % totalTasks == taskNumber`. If you are not using a shared volume " + "this could cause issues where files are not processed. Using more than one task could also affect " + "the order that the data is written to Kafka.") @PluginOwner("jcustenborder") @PluginName("kafka-connect-spooldir") @DocumentationNote("Each of the connectors in this plugin emit the following headers for each record " + "written to kafka. \n\n" + "* `file.path` - The absolute path to the file ingested.\n" + "* `file.name` - The name part of the file ingested.\n" + "* `file.name.without.extension` - The file name without the extension part of the file.\n" + "* `file.last.modified` - The last modified date of the file.\n" + "* `file.length` - The size of the file in bytes.\n" + "* `file.offset` - The offset for this piece of data within the file.\n" ) package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationNote; import com.github.jcustenborder.kafka.connect.utils.config.DocumentationWarning; import com.github.jcustenborder.kafka.connect.utils.config.Introduction; import com.github.jcustenborder.kafka.connect.utils.config.PluginName; import com.github.jcustenborder.kafka.connect.utils.config.PluginOwner; import com.github.jcustenborder.kafka.connect.utils.config.Title; ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractCleanUpPolicyTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.util.Objects; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.only; import static org.mockito.Mockito.verify; public abstract class AbstractCleanUpPolicyTest { InputFile inputFile; File inputPath; File finishedPath; File errorPath; String inputPathSubDir; protected T cleanupPolicy; protected abstract T create( InputFile inputFile, File errorPath, File finishedPath ); protected String defineInputPathSubDir() { return null; } protected ImmutableMap.Builder getConnectorConfigMap() { return new ImmutableMap.Builder() .put(SpoolDirBinaryFileSourceConnectorConfig.TOPIC_CONF, "foo") .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_PATH_CONFIG, this.inputPath.toString()) .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, "^.$") .put(SpoolDirBinaryFileSourceConnectorConfig.ERROR_PATH_CONFIG, this.errorPath.toString()) .put(SpoolDirBinaryFileSourceConnectorConfig.FINISHED_PATH_CONFIG, this.finishedPath.toString()); } @BeforeEach public void before() throws IOException { this.errorPath = Files.createTempDir(); this.finishedPath = Files.createTempDir(); this.inputPath = Files.createTempDir(); this.inputPathSubDir = defineInputPathSubDir(); File tempFileParentPathDir = this.inputPath; if (this.inputPathSubDir != null) { tempFileParentPathDir = new File(this.inputPath, this.inputPathSubDir); tempFileParentPathDir.mkdirs(); } File inputFile = File.createTempFile("input", "file", tempFileParentPathDir); SpoolDirBinaryFileSourceConnectorConfig config = new SpoolDirBinaryFileSourceConnectorConfig(getConnectorConfigMap().build()); this.inputFile = new InputFile(config, inputFile); this.inputFile.inputStreamReader = mock(InputStreamReader.class); this.inputFile.lineNumberReader = mock(LineNumberReader.class); this.cleanupPolicy = create(this.inputFile, this.errorPath, this.finishedPath); } protected File getTargetFilePath(File containerPath, InputFile inputFile) { String subDir = (this.defineInputPathSubDir() != null ? this.defineInputPathSubDir() : ""); return new File(new File(containerPath,subDir), inputFile.getName()); } @Test public void error() throws IOException { assertTrue(this.inputFile.exists(), "Input file should exist"); this.cleanupPolicy.error(); assertFalse(this.inputFile.exists(), "input file should not exist"); File erroredFile = this.getTargetFilePath(this.errorPath,this.inputFile); assertTrue(erroredFile.exists(), "errored file should exist."); } void delete(File file) { if (file.isDirectory()) { for (File child : Objects.requireNonNull(file.listFiles())) { delete(child); } } file.delete(); } @AfterEach public void after() throws IOException { delete(this.finishedPath); delete(this.errorPath); delete(this.inputPath); verify(this.inputFile.inputStreamReader, only()).close(); verify(this.inputFile.lineNumberReader, only()).close(); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSchemaGeneratorTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.Files; import org.junit.jupiter.api.BeforeEach; import java.io.File; import java.util.LinkedHashMap; import java.util.Map; public class AbstractSchemaGeneratorTest { protected Map settings; File tempRoot; File inputPath; File finishedPath; File errorPath; @BeforeEach public void createTempDir() { this.tempRoot = Files.createTempDir(); this.inputPath = new File(this.tempRoot, "input"); this.inputPath.mkdirs(); this.finishedPath = new File(this.tempRoot, "finished"); this.finishedPath.mkdirs(); this.errorPath = new File(this.tempRoot, "error"); this.errorPath.mkdirs(); this.settings = new LinkedHashMap<>(); this.settings.put(AbstractSourceConnectorConfig.INPUT_PATH_CONFIG, this.inputPath.getAbsolutePath()); this.settings.put(AbstractSourceConnectorConfig.FINISHED_PATH_CONFIG, this.finishedPath.getAbsolutePath()); this.settings.put(AbstractSourceConnectorConfig.ERROR_PATH_CONFIG, this.errorPath.getAbsolutePath()); this.settings.put(AbstractSourceConnectorConfig.TOPIC_CONF, "dummy"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSpoolDirSourceConnectorTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.Files; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.LinkedHashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertNotNull; public abstract class AbstractSpoolDirSourceConnectorTest { private static final Logger log = LoggerFactory.getLogger(AbstractSpoolDirSourceConnectorTest.class); protected T connector; protected Map settings; File tempRoot; File inputPath; File finishedPath; File errorPath; protected abstract T createConnector(); @BeforeEach public void before() { this.connector = createConnector(); } @Test public void taskClass() { assertNotNull(this.connector.taskClass()); } @BeforeEach public void createTempDir() { this.tempRoot = Files.createTempDir(); this.inputPath = new File(this.tempRoot, "input"); this.inputPath.mkdirs(); this.finishedPath = new File(this.tempRoot, "finished"); this.finishedPath.mkdirs(); this.errorPath = new File(this.tempRoot, "error"); this.errorPath.mkdirs(); this.settings = new LinkedHashMap<>(); this.settings.put(AbstractSourceConnectorConfig.INPUT_PATH_CONFIG, this.inputPath.getAbsolutePath()); this.settings.put(AbstractSourceConnectorConfig.FINISHED_PATH_CONFIG, this.finishedPath.getAbsolutePath()); this.settings.put(AbstractSourceConnectorConfig.ERROR_PATH_CONFIG, this.errorPath.getAbsolutePath()); this.settings.put(AbstractSourceConnectorConfig.TOPIC_CONF, "dummy"); this.settings.put(AbstractSpoolDirSourceConnectorConfig.SCHEMA_GENERATION_ENABLED_CONF, "true"); } @AfterEach public void cleanupTempDir() throws IOException { java.nio.file.Files.walkFileTree(this.tempRoot.toPath(), new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { log.trace("cleanupTempDir() - Removing {}", file); java.nio.file.Files.delete(file); return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { log.trace("cleanupTempDir() - Removing {}", file); java.nio.file.Files.delete(file); return FileVisitResult.CONTINUE; } }); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/AbstractSpoolDirSourceTaskTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.fasterxml.jackson.databind.SerializationFeature; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import com.google.common.collect.Maps; import com.google.common.io.ByteStreams; import com.google.common.io.Files; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTaskContext; import org.apache.kafka.connect.storage.OffsetStorageReader; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import static com.github.jcustenborder.kafka.connect.utils.AssertConnectRecord.assertSourceRecord; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public abstract class AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(AbstractSpoolDirSourceTaskTest.class); protected File tempDirectory; protected File inputPath; protected File errorPath; protected File finishedPath; protected T task; @BeforeEach public void setup() { this.tempDirectory = Files.createTempDir(); this.finishedPath = new File(this.tempDirectory, "finished"); this.inputPath = new File(this.tempDirectory, "input"); this.errorPath = new File(this.tempDirectory, "error"); this.finishedPath.mkdirs(); this.inputPath.mkdirs(); this.errorPath.mkdirs(); } @BeforeEach public void configureIndent() { ObjectMapperFactory.INSTANCE.configure(SerializationFeature.INDENT_OUTPUT, true); } protected abstract T createTask(); protected Map settings() { Map settings = Maps.newLinkedHashMap(); settings.put(AbstractSourceConnectorConfig.INPUT_PATH_CONFIG, this.inputPath.getAbsolutePath()); settings.put(AbstractSourceConnectorConfig.FINISHED_PATH_CONFIG, this.finishedPath.getAbsolutePath()); settings.put(AbstractSourceConnectorConfig.ERROR_PATH_CONFIG, this.errorPath.getAbsolutePath()); settings.put(AbstractSourceConnectorConfig.TOPIC_CONF, "testing"); settings.put(AbstractSourceConnectorConfig.EMPTY_POLL_WAIT_MS_CONF, "10"); return settings; } protected String defineInputPathSubDir() { return null; } protected File getTargetFilePath(File containerPath, String inputFileName) { String subDir = (this.defineInputPathSubDir() != null ? this.defineInputPathSubDir() : ""); File targetDir = new File(containerPath, subDir); targetDir.mkdirs(); return new File(targetDir, inputFileName); } protected void poll(final String packageName, TestCase testCase) throws InterruptedException, IOException { String keySchemaConfig = ObjectMapperFactory.INSTANCE.writeValueAsString(testCase.keySchema); String valueSchemaConfig = ObjectMapperFactory.INSTANCE.writeValueAsString(testCase.valueSchema); Map settings = this.settings(); settings.put(AbstractSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, String.format("^.*\\.%s", packageName)); settings.put(AbstractSpoolDirSourceConnectorConfig.KEY_SCHEMA_CONF, keySchemaConfig); settings.put(AbstractSpoolDirSourceConnectorConfig.VALUE_SCHEMA_CONF, valueSchemaConfig); if (null != testCase.settings && !testCase.settings.isEmpty()) { settings.putAll(testCase.settings); } this.task = createTask(); SourceTaskContext sourceTaskContext = mock(SourceTaskContext.class); OffsetStorageReader offsetStorageReader = mock(OffsetStorageReader.class); when(offsetStorageReader.offset(anyMap())).thenReturn(testCase.offset); when(sourceTaskContext.offsetStorageReader()).thenReturn(offsetStorageReader); this.task.initialize(sourceTaskContext); this.task.start(settings); String dataFile = new File(packageName, Files.getNameWithoutExtension(testCase.path.toString())) + ".data"; log.trace("poll(String, TestCase) - dataFile={}", dataFile); String inputFileName = String.format("%s.%s", Files.getNameWithoutExtension(testCase.path.toString()), packageName ); //Use this config because it's the simplest. SpoolDirBinaryFileSourceConnectorConfig config = new SpoolDirBinaryFileSourceConnectorConfig(settings); final File p = this.getTargetFilePath(this.inputPath, inputFileName); try (InputStream inputStream = this.getClass().getResourceAsStream(dataFile)) { assertNotNull( inputStream, String.format("Resource stream '%s' was not found", dataFile) ); try (OutputStream outputStream = new FileOutputStream(p)) { ByteStreams.copy(inputStream, outputStream); } } final InputFile inputFile = new InputFile(config, p); log.trace("poll(String, TestCase) - inputFile = {}", inputFile); assertFalse(inputFile.processingFlag().exists(), String.format("processingFile %s should not exist before first poll().", inputFile.processingFlag())); assertTrue(inputFile.exists(), String.format("inputFile %s should exist.", inputFile)); List records = this.task.poll(); assertTrue(inputFile.exists(), String.format("inputFile %s should exist after first poll().", inputFile)); assertTrue(inputFile.processingFlag().exists(), String.format("processingFile %s should exist after first poll().", inputFile.processingFlag())); assertNotNull(records, "records should not be null."); assertFalse(records.isEmpty(), "records should not be empty"); assertEquals(testCase.expected.size(), records.size(), "records.size() does not match."); /* The following headers will change. Lets ensure they are there but we don't care about their values since they are driven by things that will change such as lastModified dates and paths. */ List headersToRemove = new ArrayList(); headersToRemove.add(Metadata.HEADER_LAST_MODIFIED); headersToRemove.add(Metadata.HEADER_PATH); headersToRemove.add(Metadata.HEADER_LENGTH); headersToRemove.add(Metadata.HEADER_NAME_WITHOUT_EXTENSION); headersToRemove.add(Metadata.HEADER_PARENT_DIR_NAME); headersToRemove.add(Metadata.HEADER_FILE_RELATIVE_PATH); for (int i = 0; i < testCase.expected.size(); i++) { SourceRecord expectedRecord = testCase.expected.get(i); SourceRecord actualRecord = records.get(i); for (String headerToRemove : headersToRemove) { assertNotNull( actualRecord.headers().lastWithName(headerToRemove), String.format("index:%s should have the header '%s'", i, headerToRemove) ); actualRecord.headers().remove(headerToRemove); expectedRecord.headers().remove(headerToRemove); } assertSourceRecord(expectedRecord, actualRecord, String.format("index:%s", i)); } records = this.task.poll(); assertNull(records, "records should be null after first poll."); records = this.task.poll(); assertNull(records, "records should be null after first poll."); assertFalse(inputFile.exists(), String.format("inputFile %s should not exist.", inputFile)); assertFalse(inputFile.processingFlag().exists(), String.format("processingFile %s should not exist.", inputFile.processingFlag())); final File finishedFile = this.getTargetFilePath(this.finishedPath, inputFileName); assertTrue(finishedFile.exists(), String.format("finishedFile %s should exist.", finishedFile)); } protected List loadTestCases(String packageName) throws IOException { String packagePrefix = String.format( "%s.%s", this.getClass().getPackage().getName(), packageName ); log.trace("packagePrefix = {}", packagePrefix); List testCases = TestDataUtils.loadJsonResourceFiles(packagePrefix, TestCase.class); if (testCases.isEmpty() && log.isWarnEnabled()) { log.warn("No test cases were found in the resources. packagePrefix = {}", packagePrefix); } return testCases; } @Test public void version() { this.task = createTask(); assertNotNull(this.task.version(), "version should not be null."); } @Test public void recordOffsetNPE() { this.task = createTask(); long actual = this.task.recordOffset(); } @AfterEach public void after() { if (null != this.task) { this.task.stop(); } } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/ByNameAbstractTaskPartitionerPredicateTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import java.io.File; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.UUID; import java.util.stream.IntStream; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.DynamicTest.dynamicTest; public class ByNameAbstractTaskPartitionerPredicateTest { protected List input; @TestFactory public Stream test() { List files = new ArrayList<>(500); for (int i = 0; i < 500; i++) { files.add(new File(UUID.randomUUID().toString())); } return IntStream.range(2, 50).boxed().map(count -> dynamicTest(count.toString(), () -> { Set queue = new LinkedHashSet<>(files); for (int index = 0; index <= count; index++) { AbstractTaskPartitionerPredicate.ByName predicate = new AbstractTaskPartitionerPredicate.ByName(index, count); files.stream() .filter(predicate) .forEach(queue::remove); } assertEquals(0, queue.size(), "Queue should be empty"); })); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/CsvSchemaGeneratorTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Map; import static com.github.jcustenborder.kafka.connect.utils.AssertSchema.assertSchema; public class CsvSchemaGeneratorTest extends AbstractSchemaGeneratorTest { @Test public void foo() throws IOException { File inputFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.data"); this.settings.put(SpoolDirCsvSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF, "true"); CsvSchemaGenerator schemaGenerator = new CsvSchemaGenerator(settings); Map.Entry kvp = schemaGenerator.generate(inputFile, Arrays.asList("id")); final Schema expectedKeySchema = SchemaBuilder.struct() .name("com.github.jcustenborder.kafka.connect.model.Key") .field("id", Schema.OPTIONAL_STRING_SCHEMA) .build(); final Schema expectedValueSchema = SchemaBuilder.struct() .name("com.github.jcustenborder.kafka.connect.model.Value") .field("id", Schema.OPTIONAL_STRING_SCHEMA) .field("first_name", Schema.OPTIONAL_STRING_SCHEMA) .field("last_name", Schema.OPTIONAL_STRING_SCHEMA) .field("email", Schema.OPTIONAL_STRING_SCHEMA) .field("gender", Schema.OPTIONAL_STRING_SCHEMA) .field("ip_address", Schema.OPTIONAL_STRING_SCHEMA) .field("last_login", Schema.OPTIONAL_STRING_SCHEMA) .field("account_balance", Schema.OPTIONAL_STRING_SCHEMA) .field("country", Schema.OPTIONAL_STRING_SCHEMA) .field("favorite_color", Schema.OPTIONAL_STRING_SCHEMA) .build(); assertSchema(expectedKeySchema, kvp.getKey(), "key schema does not match."); assertSchema(expectedValueSchema, kvp.getValue(), "value schema does not match."); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/DeleteCleanupPolicySubDirsNoRetainTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; public class DeleteCleanupPolicySubDirsNoRetainTest extends DeleteCleanupPolicyTest { @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } protected ImmutableMap.Builder getConnectorConfigMap() { return super.getConnectorConfigMap() .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY, "true") .put(SpoolDirBinaryFileSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH, "false"); } @Test public void success() throws IOException { super.success(); assertFalse(new File(this.inputPath,this.defineInputPathSubDir()).exists(), "The input.path sub-directory "+this.defineInputPathSubDir()+" should not exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/DeleteCleanupPolicySubDirsRetainTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertTrue; public class DeleteCleanupPolicySubDirsRetainTest extends DeleteCleanupPolicyTest { @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } protected ImmutableMap.Builder getConnectorConfigMap() { return super.getConnectorConfigMap() .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY, "true") .put(SpoolDirBinaryFileSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH, "true"); } @Test public void success() throws IOException { super.success(); assertTrue(new File(this.inputPath,this.defineInputPathSubDir()).exists(), "The input.path sub-directory "+this.defineInputPathSubDir()+" should exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/DeleteCleanupPolicyTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class DeleteCleanupPolicyTest extends AbstractCleanUpPolicyTest { @Override protected AbstractCleanUpPolicy.Delete create(InputFile inputFile, File errorPath, File finishedPath) { return new AbstractCleanUpPolicy.Delete(inputFile, errorPath, finishedPath); } @Test public void success() throws IOException { assertTrue(this.inputFile.exists(), "Input file should exist"); this.cleanupPolicy.success(); assertFalse(this.inputFile.exists(), "Input file should not exist"); if (!(cleanupPolicy instanceof AbstractCleanUpPolicy.Delete)) { File finishedFile = new File(this.finishedPath, this.inputFile.getName()); assertTrue(finishedFile.exists(), "finishedPath file should exist."); } else { File finishedFile = new File(this.finishedPath, this.inputFile.getName()); assertFalse(finishedFile.exists(), "finishedPath file should exist."); } } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/DocumentationTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.BaseDocumentationTest; public class DocumentationTest extends BaseDocumentationTest { } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/FileComparatorTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.collect.ImmutableList; import com.google.common.io.Files; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; public class FileComparatorTest { File tempDirectory; @BeforeEach public void before() { this.tempDirectory = Files.createTempDir(); } File createFile(String name) throws IOException { return createFile(name, new Date().getTime(), 0); } File createFile(String name, long date) throws IOException { return createFile(name, date, 0); } File createFile(String name, long date, long length) throws IOException { File result = new File(tempDirectory, name); if (length == 0) { Files.touch(result); } else { Files.write( new byte[(int) length], result ); } result.setLastModified(date); return result; } List sort(List files, AbstractSourceConnectorConfig.FileAttribute... attributes) { List result = new ArrayList<>(files); FileComparator comparator = new FileComparator(ImmutableList.copyOf(attributes)); result.sort(comparator); return result; } List expected(List files, int... indexes) { List result = new ArrayList<>(); for (int index : indexes) { result.add(files.get(index)); } return result; } @Test public void existingFunctionality() throws IOException { List input = Arrays.asList( createFile("File1.csv"), createFile("File2.csv"), createFile("File3.csv") ); List expected = expected(input, 0, 1, 2); List actual = sort(input, AbstractSourceConnectorConfig.FileAttribute.NameAsc); assertEquals(expected, actual); } @Test public void sortByLastModified() throws IOException { long lastModified = new Date().getTime(); List input = Arrays.asList( createFile("File1.csv", lastModified-=1000L), createFile("File2.csv", lastModified-=1000L), createFile("File3.csv", lastModified-=1000L) ); List expected = expected(input, 2, 1, 0); List actual = sort(input, AbstractSourceConnectorConfig.FileAttribute.LastModifiedAsc); assertEquals(expected, actual); } @Test public void sortBySize() throws IOException { long lastModified = new Date().getTime(); long length = 10000; List input = Arrays.asList( createFile("File1.csv", lastModified-=1000L, length-=1000L), createFile("File2.csv", lastModified-=1000L, length-=1000L), createFile("File3.csv", lastModified-=1000L, length-=1000L) ); List expected = expected(input, 0, 1, 2); List actual = sort(input, AbstractSourceConnectorConfig.FileAttribute.LengthDesc); assertEquals(expected, actual); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/JsonSchemaGeneratorTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import java.util.HashMap; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Map; import static com.github.jcustenborder.kafka.connect.utils.AssertSchema.assertSchema; public class JsonSchemaGeneratorTest extends AbstractSchemaGeneratorTest { @Test public void schema() throws IOException { File inputFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FieldsMatch.data"); JsonSchemaGenerator schemaGenerator = new JsonSchemaGenerator(settings); Map.Entry kvp = schemaGenerator.generate(inputFile, Arrays.asList("id")); final Schema expectedKeySchema = SchemaBuilder.struct() .name("com.github.jcustenborder.kafka.connect.model.Key") .field("id", Schema.OPTIONAL_STRING_SCHEMA) .build(); final Schema expectedValueSchema = SchemaBuilder.struct() .name("com.github.jcustenborder.kafka.connect.model.Value") .field("id", Schema.OPTIONAL_STRING_SCHEMA) .field("first_name", Schema.OPTIONAL_STRING_SCHEMA) .field("last_name", Schema.OPTIONAL_STRING_SCHEMA) .field("email", Schema.OPTIONAL_STRING_SCHEMA) .field("gender", Schema.OPTIONAL_STRING_SCHEMA) .field("ip_address", Schema.OPTIONAL_STRING_SCHEMA) .field("last_login", Schema.OPTIONAL_STRING_SCHEMA) .field("account_balance", Schema.OPTIONAL_STRING_SCHEMA) .field("country", Schema.OPTIONAL_STRING_SCHEMA) .field("favorite_color", Schema.OPTIONAL_STRING_SCHEMA) .build(); assertSchema(expectedKeySchema, kvp.getKey(), "key schema does not match."); assertSchema(expectedValueSchema, kvp.getValue(), "value schema does not match."); } @Test public void schemaWithCustomSchemaName() throws IOException { File inputFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FieldsMatch.data"); Map configs = new HashMap<>(settings); configs.put(AbstractSpoolDirSourceConnectorConfig.SCHEMA_GENERATION_KEY_NAME_CONF, "com.foo.key"); configs.put(AbstractSpoolDirSourceConnectorConfig.SCHEMA_GENERATION_VALUE_NAME_CONF, "com.foo.value"); JsonSchemaGenerator schemaGenerator = new JsonSchemaGenerator(configs); Map.Entry kvp = schemaGenerator.generate(inputFile, Arrays.asList("id")); final Schema expectedKeySchema = SchemaBuilder.struct() .name("com.foo.key") .field("id", Schema.OPTIONAL_STRING_SCHEMA) .build(); final Schema expectedValueSchema = SchemaBuilder.struct() .name("com.foo.value") .field("id", Schema.OPTIONAL_STRING_SCHEMA) .field("first_name", Schema.OPTIONAL_STRING_SCHEMA) .field("last_name", Schema.OPTIONAL_STRING_SCHEMA) .field("email", Schema.OPTIONAL_STRING_SCHEMA) .field("gender", Schema.OPTIONAL_STRING_SCHEMA) .field("ip_address", Schema.OPTIONAL_STRING_SCHEMA) .field("last_login", Schema.OPTIONAL_STRING_SCHEMA) .field("account_balance", Schema.OPTIONAL_STRING_SCHEMA) .field("country", Schema.OPTIONAL_STRING_SCHEMA) .field("favorite_color", Schema.OPTIONAL_STRING_SCHEMA) .build(); assertSchema(expectedKeySchema, kvp.getKey(), "key schema does not match."); assertSchema(expectedValueSchema, kvp.getValue(), "value schema does not match."); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/MinimumFileAgePredicateTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.apache.kafka.common.utils.Time; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class MinimumFileAgePredicateTest { File inputFile; static final String EXTENSION = "processing"; @BeforeEach public void before() throws IOException { this.inputFile = File.createTempFile("test", "file"); } @AfterEach public void after() throws IOException { if (null != this.inputFile && this.inputFile.exists()) { this.inputFile.delete(); } } Time time(long milliseconds) { Time time = mock(Time.class); when(time.milliseconds()).thenReturn(milliseconds); return time; } @Test public void notOldEnough() throws IOException { long timestamp = 1559653835123L; Time time = time(timestamp); this.inputFile.setLastModified(timestamp); InputFileDequeue.MinimumFileAgePredicate predicate = new InputFileDequeue.MinimumFileAgePredicate( 1000, time ); assertFalse(predicate.test(this.inputFile), "File should not be old enough"); } @Test public void oldEnough() throws IOException { long timestamp = 1559653835123L; this.inputFile.setLastModified(timestamp); timestamp += 5000L; Time time = time(timestamp); InputFileDequeue.MinimumFileAgePredicate predicate = new InputFileDequeue.MinimumFileAgePredicate( 1000, time ); assertTrue(predicate.test(this.inputFile), "File should be old enough"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/MoveByDateCleanupPolicySubDirsNoRetainTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; public class MoveByDateCleanupPolicySubDirsNoRetainTest extends MoveByDateCleanupPolicyTest { @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } protected ImmutableMap.Builder getConnectorConfigMap() { return super.getConnectorConfigMap() .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY, "true") .put(SpoolDirBinaryFileSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH, "false"); } @Test public void success() throws IOException { super.success(); assertFalse(new File(this.inputPath,this.defineInputPathSubDir()).exists(), "The input.path sub-directory "+this.defineInputPathSubDir()+" should not exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/MoveByDateCleanupPolicySubDirsRetainTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertTrue; public class MoveByDateCleanupPolicySubDirsRetainTest extends MoveByDateCleanupPolicyTest { @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } protected ImmutableMap.Builder getConnectorConfigMap() { return super.getConnectorConfigMap() .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY, "true") .put(SpoolDirBinaryFileSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH, "true"); } @Test public void success() throws IOException { super.success(); assertTrue(new File(this.inputPath,this.defineInputPathSubDir()).exists(), "The input.path sub-directory "+this.defineInputPathSubDir()+" should exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/MoveByDateCleanupPolicyTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class MoveByDateCleanupPolicyTest extends AbstractCleanUpPolicyTest { @Override protected AbstractCleanUpPolicy.MoveByDate create(InputFile inputFile, File errorPath, File finishedPath) { return new AbstractCleanUpPolicy.MoveByDate(inputFile, errorPath, finishedPath); } @Test public void success() throws IOException { SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd"); Path subDirectory = Paths.get(this.finishedPath.getAbsolutePath(), dateFormatter.format(this.inputFile.lastModified())); File finishedFile = this.getTargetFilePath(subDirectory.toFile(), this.inputFile); assertTrue(this.inputFile.exists(), "Input file should exist"); assertFalse(finishedFile.exists(), "Finished file should not exist"); this.cleanupPolicy.success(); assertFalse(this.inputFile.exists(), "Input file should not exist"); assertTrue(finishedFile.exists(), "Finished file should exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/MoveCleanupPolicySubDirsNoRetainTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; public class MoveCleanupPolicySubDirsNoRetainTest extends MoveCleanupPolicyTest { @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } protected ImmutableMap.Builder getConnectorConfigMap() { return super.getConnectorConfigMap() .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY, "true") .put(SpoolDirBinaryFileSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH, "false"); } @Test public void success() throws IOException { super.success(); assertFalse(new File(this.inputPath,this.defineInputPathSubDir()).exists(), "The input.path sub-directory "+this.defineInputPathSubDir()+" should not exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/MoveCleanupPolicySubDirsRetainTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertTrue; public class MoveCleanupPolicySubDirsRetainTest extends MoveCleanupPolicyTest { @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } protected ImmutableMap.Builder getConnectorConfigMap() { return super.getConnectorConfigMap() .put(SpoolDirBinaryFileSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY, "true") .put(SpoolDirBinaryFileSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH, "true"); } @Test public void success() throws IOException { super.success(); assertTrue(new File(this.inputPath,this.defineInputPathSubDir()).exists(), "The input.path sub-directory "+this.defineInputPathSubDir()+" should exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/MoveCleanupPolicyTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class MoveCleanupPolicyTest extends AbstractCleanUpPolicyTest { @Override protected AbstractCleanUpPolicy.Move create(InputFile inputFile, File errorPath, File finishedPath) { return new AbstractCleanUpPolicy.Move(inputFile, errorPath, finishedPath); } @Test public void success() throws IOException { File finishedFile = this.getTargetFilePath(this.finishedPath, this.inputFile); assertTrue(this.inputFile.exists(), "Input file should exist"); assertFalse(finishedFile.exists(), "Finished file should not exist"); this.cleanupPolicy.success(); assertFalse(this.inputFile.exists(), "Input file should not exist"); assertTrue(finishedFile.exists(), "Finished file should exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/NamedTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import java.nio.file.Path; public interface NamedTest { void path(Path path); } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/NoneCleanupPolicyTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertTrue; public class NoneCleanupPolicyTest extends AbstractCleanUpPolicyTest { @Override protected AbstractCleanUpPolicy.None create(InputFile inputFile, File errorPath, File finishedPath) { return new AbstractCleanUpPolicy.None(inputFile, errorPath, finishedPath); } @Test public void success() throws IOException { assertTrue(this.inputFile.exists(), "Input file should exist"); this.cleanupPolicy.success(); assertTrue(this.inputFile.exists(), "Input file should exist"); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/ProcessingFileExistsPredicateTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.Files; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class ProcessingFileExistsPredicateTest { File inputFile; InputFileDequeue.ProcessingFileExistsPredicate predicate; static final String EXTENSION = "processing"; @BeforeEach public void before() throws IOException { this.inputFile = File.createTempFile("test", "file"); this.predicate = new InputFileDequeue.ProcessingFileExistsPredicate(EXTENSION); } @AfterEach public void after() throws IOException { if (null != this.inputFile && this.inputFile.exists()) { this.inputFile.delete(); } } @Test public void test() throws IOException { File processingFlag = InputFileDequeue.processingFile(EXTENSION, this.inputFile); Files.touch(processingFlag); assertFalse(this.predicate.test(this.inputFile)); processingFlag.delete(); assertTrue(this.predicate.test(this.inputFile)); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirAvroSourceTaskTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.Files; import io.confluent.connect.avro.AvroData; import org.apache.avro.Schema; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileWriter; import org.apache.avro.generic.GenericContainer; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.io.DatumWriter; import org.apache.kafka.connect.source.SourceRecord; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.DynamicTest.dynamicTest; public class SpoolDirAvroSourceTaskTest extends AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirJsonSourceTaskTest.class); @Override protected SpoolDirAvroSourceTask createTask() { return new SpoolDirAvroSourceTask(); } @Override protected Map settings() { Map settings = super.settings(); return settings; } @Disabled @Test public void foo() throws IOException { File outputFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/avro/FieldsMatch.data"); DatumWriter datumWriter = new GenericDatumWriter<>(); DataFileWriter writer = new DataFileWriter<>(datumWriter); writer.setCodec(CodecFactory.bzip2Codec()); final String packageName = "csv"; List testCases = loadTestCases(packageName); Optional testcase = testCases.stream().filter(testCase -> testCase.path.getFileName().endsWith("FieldsMatch.json")).findFirst(); assertTrue(testcase.isPresent()); AvroData avroData = new AvroData(1235); Schema schema = null; for (SourceRecord expected : testcase.get().expected) { if (null == schema) { schema = avroData.fromConnectSchema(expected.valueSchema()); writer.create(schema, outputFile); } GenericContainer value = (GenericContainer) avroData.fromConnectData(expected.valueSchema(), expected.value()); writer.append(value); } writer.close(); } @TestFactory public Stream poll() throws IOException { final String packageName = "avro"; List testCases = loadTestCases(packageName); return testCases.stream().map(testCase -> { String name = Files.getNameWithoutExtension(testCase.path.toString()); return dynamicTest(name, () -> { poll(packageName, testCase); }); }); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirBinaryFileSourceTaskTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.Files; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicTest.dynamicTest; public class SpoolDirBinaryFileSourceTaskTest extends AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirJsonSourceTaskTest.class); @Override protected SpoolDirBinaryFileSourceTask createTask() { return new SpoolDirBinaryFileSourceTask(); } @Override protected Map settings() { Map settings = super.settings(); return settings; } @TestFactory public Stream poll() throws IOException { final String packageName = "binary"; List testCases = loadTestCases(packageName); return testCases.stream().map(testCase -> { String name = Files.getNameWithoutExtension(testCase.path.toString()); return dynamicTest(name, () -> { poll(packageName, testCase); }); }); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnectorConfigTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; import com.opencsv.ICSVParser; import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.StringReader; import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertArrayEquals; public class SpoolDirCsvSourceConnectorConfigTest { @Test public void nullFieldSeparator() throws IOException { Map settings = new HashMap<>(); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_SEPARATOR_CHAR_CONF, "0"); settings.put(SpoolDirCsvSourceConnectorConfig.TOPIC_CONF, "test"); settings.put(SpoolDirCsvSourceConnectorConfig.INPUT_PATH_CONFIG, "/tmp"); settings.put(SpoolDirCsvSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, "^.+$"); settings.put(SpoolDirCsvSourceConnectorConfig.ERROR_PATH_CONFIG, "/tmp"); settings.put(SpoolDirCsvSourceConnectorConfig.FINISHED_PATH_CONFIG, "/tmp"); settings.put(SpoolDirCsvSourceConnectorConfig.KEY_SCHEMA_CONF, "{\n" + " \"name\" : \"com.example.users.UserKey\",\n" + " \"type\" : \"STRUCT\",\n" + " \"isOptional\" : false,\n" + " \"fieldSchemas\" : {\n" + " \"id\" : {\n" + " \"type\" : \"INT64\",\n" + " \"isOptional\" : false\n" + " }\n" + " }\n" + " }"); settings.put(SpoolDirCsvSourceConnectorConfig.VALUE_SCHEMA_CONF, "{\n" + " \"name\" : \"com.example.users.UserKey\",\n" + " \"type\" : \"STRUCT\",\n" + " \"isOptional\" : false,\n" + " \"fieldSchemas\" : {\n" + " \"id\" : {\n" + " \"type\" : \"INT64\",\n" + " \"isOptional\" : false\n" + " }\n" + " }\n" + " }"); SpoolDirCsvSourceConnectorConfig config = new SpoolDirCsvSourceConnectorConfig( true, settings ); ICSVParser parser = config.createCSVParserBuilder(); try (StringReader reader = new StringReader("id\u0000test\n123\u0000foo")) { CSVReaderBuilder readerBuilder = config.createCSVReaderBuilder(reader, parser); try (CSVReader csvReader = readerBuilder.build()) { String[] line = csvReader.readNext(); assertArrayEquals(new String[]{"id", "test"}, line); line = csvReader.readNext(); assertArrayEquals(new String[]{"123", "foo"}, line); } } } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnectorTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.ByteStreams; import org.apache.kafka.connect.errors.DataException; import org.junit.jupiter.api.Test; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import static org.junit.jupiter.api.Assertions.assertThrows; public class SpoolDirCsvSourceConnectorTest extends AbstractSpoolDirSourceConnectorTest { @Override protected SpoolDirCsvSourceConnector createConnector() { return new SpoolDirCsvSourceConnector(); } @Test public void startWithoutSchema() throws IOException { this.settings.put(AbstractSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, "^.*\\.csv$"); String[] inputFiles = new String[]{ "csv/FieldsMatch.data", "csv/FieldsMatch.data", }; int index = 0; for (String inputFile : inputFiles) { try (InputStream inputStream = this.getClass().getResourceAsStream(inputFile)) { File outputFile = new File(this.inputPath, "input" + index + ".csv"); try (OutputStream outputStream = new FileOutputStream(outputFile)) { ByteStreams.copy(inputStream, outputStream); } } index++; } this.connector.start(settings); } @Test() public void startWithoutSchemaMismatch() throws IOException { this.settings.put(AbstractSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, "^.*\\.csv$"); String[] inputFiles = new String[]{ "csv/FieldsMatch.data", "csv/DataHasMoreFields.data", }; int index = 0; for (String inputFile : inputFiles) { try (InputStream inputStream = this.getClass().getResourceAsStream(inputFile)) { File outputFile = new File(this.inputPath, "input" + index + ".csv"); try (OutputStream outputStream = new FileOutputStream(outputFile)) { ByteStreams.copy(inputStream, outputStream); } } index++; } assertThrows(DataException.class, () -> { this.connector.start(settings); }); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTaskSubDirsNoRetainTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; public class SpoolDirCsvSourceTaskSubDirsNoRetainTest extends SpoolDirCsvSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirCsvSourceTaskSubDirsNoRetainTest.class); @Override protected Map settings() { Map settings = super.settings(); settings.put(AbstractSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY,"true"); settings.put(AbstractSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH,"false"); return settings; } @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTaskSubDirsRetainTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; public class SpoolDirCsvSourceTaskSubDirsRetainTest extends SpoolDirCsvSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirCsvSourceTaskSubDirsRetainTest.class); @Override protected Map settings() { Map settings = super.settings(); settings.put(AbstractSourceConnectorConfig.INPUT_PATH_WALK_RECURSIVELY,"true"); settings.put(AbstractSourceConnectorConfig.CLEANUP_POLICY_MAINTAIN_RELATIVE_PATH,"true"); return settings; } @Override protected String defineInputPathSubDir() { return "test/01/02/03"; } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTaskTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import com.google.common.io.Files; import com.opencsv.CSVWriterBuilder; import com.opencsv.ICSVWriter; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTaskContext; import org.apache.kafka.connect.storage.OffsetStorageReader; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Writer; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.DynamicTest.dynamicTest; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class SpoolDirCsvSourceTaskTest extends AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirCsvSourceTaskTest.class); @Override protected SpoolDirCsvSourceTask createTask() { return new SpoolDirCsvSourceTask(); } @Override protected Map settings() { Map settings = super.settings(); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF, "true"); settings.put(SpoolDirCsvSourceConnectorConfig.PARSER_TIMESTAMP_DATE_FORMATS_CONF, "yyyy-MM-dd'T'HH:mm:ss'Z'"); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_NULL_FIELD_INDICATOR_CONF, "BOTH"); return settings; } @TestFactory public Stream poll() throws IOException { final String packageName = "csv"; List testCases = loadTestCases(packageName); return testCases.stream().map(testCase -> { String name = Files.getNameWithoutExtension(testCase.path.toString()); return dynamicTest(name, () -> { poll(packageName, testCase); }); }); } void writeCSV(File outputFile, Schema schema, List structs) throws IOException { try (Writer writer = new FileWriter(outputFile)) { try (ICSVWriter csvWriter = new CSVWriterBuilder(writer) .build()) { String[] header = schema.fields().stream().map(Field::name).toArray(String[]::new); csvWriter.writeNext(header); for (Struct struct : structs) { List values = new ArrayList<>(); for (Field field : schema.fields()) { values.add(struct.get(field).toString()); } csvWriter.writeNext(values.toArray(new String[0])); } csvWriter.flush(); } } } @Test public void rebalance() throws IOException, InterruptedException { Schema schema = SchemaBuilder.struct() .field("id", Schema.INT32_SCHEMA) .build(); final int count = 100; List values = new ArrayList<>(count); for (int i = 0; i < count; i++) { values.add( new Struct(schema) .put("id", i) ); } File inputFile = this.getTargetFilePath(this.inputPath, "input.csv"); writeCSV(inputFile, schema, values); Map settings = settings(); settings.put(SpoolDirCsvSourceConnectorConfig.KEY_SCHEMA_CONF, ObjectMapperFactory.INSTANCE.writeValueAsString(schema)); settings.put(SpoolDirCsvSourceConnectorConfig.VALUE_SCHEMA_CONF, ObjectMapperFactory.INSTANCE.writeValueAsString(schema)); settings.put(SpoolDirCsvSourceConnectorConfig.BATCH_SIZE_CONF, "50"); settings.put(SpoolDirCsvSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, ".*"); SpoolDirCsvSourceTask task = new SpoolDirCsvSourceTask(); SourceTaskContext sourceTaskContext = mock(SourceTaskContext.class); OffsetStorageReader offsetStorageReader = mock(OffsetStorageReader.class); when(offsetStorageReader.offset(anyMap())) .thenReturn(null); when(sourceTaskContext.offsetStorageReader()).thenReturn(offsetStorageReader); task.initialize(sourceTaskContext); task.start(settings); List records = new ArrayList<>(); records.addAll(task.poll()); assertEquals(50, records.size()); SourceRecord lastRecord = records.get(49); when(offsetStorageReader.offset(anyMap())).thenReturn((Map) lastRecord.sourceOffset()); task.stop(); task.start(settings); records.addAll(task.poll()); assertEquals(count, records.size(), "Expected number of records does not match."); assertNull(task.poll(), "Polling should be finished with the file by now."); assertNull(task.poll(), "Polling should be finished with the file by now."); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnectorTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.ByteStreams; import org.apache.kafka.connect.errors.DataException; import org.junit.jupiter.api.Test; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import static org.junit.jupiter.api.Assertions.assertThrows; public class SpoolDirJsonSourceConnectorTest extends AbstractSpoolDirSourceConnectorTest { @Override protected SpoolDirJsonSourceConnector createConnector() { return new SpoolDirJsonSourceConnector(); } @Test public void startWithoutSchema() throws IOException { settings.put(AbstractSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, "^.*\\.json$"); String[] inputFiles = new String[]{ "json/FieldsMatch.data", "json/FieldsMatch.data", }; int index = 0; for (String inputFile : inputFiles) { try (InputStream inputStream = this.getClass().getResourceAsStream(inputFile)) { File outputFile = new File(this.inputPath, "input" + index + ".json"); try (OutputStream outputStream = new FileOutputStream(outputFile)) { ByteStreams.copy(inputStream, outputStream); } } index++; } this.connector.start(settings); } @Test() public void startWithoutSchemaMismatch() throws IOException { this.settings.put(AbstractSourceConnectorConfig.INPUT_FILE_PATTERN_CONF, "^.*\\.json$"); String[] inputFiles = new String[]{ "json/FieldsMatch.data", "json/DataHasMoreFields.data", }; int index = 0; for (String inputFile : inputFiles) { try (InputStream inputStream = this.getClass().getResourceAsStream(inputFile)) { File outputFile = new File(this.inputPath, "input" + index + ".json"); try (OutputStream outputStream = new FileOutputStream(outputFile)) { ByteStreams.copy(inputStream, outputStream); } } index++; } assertThrows(DataException.class, () -> { this.connector.start(settings); }); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceTaskTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.Files; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicTest.dynamicTest; public class SpoolDirJsonSourceTaskTest extends AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirJsonSourceTaskTest.class); @Override protected SpoolDirJsonSourceTask createTask() { return new SpoolDirJsonSourceTask(); } @Override protected Map settings() { Map settings = super.settings(); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF, "true"); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_NULL_FIELD_INDICATOR_CONF, "BOTH"); settings.put(SpoolDirCsvSourceConnectorConfig.PARSER_TIMESTAMP_DATE_FORMATS_CONF, "yyyy-MM-dd'T'HH:mm:ss'Z'"); return settings; } @TestFactory public Stream poll() throws IOException { final String packageName = "json"; List testCases = loadTestCases(packageName); return testCases.stream().map(testCase -> { String name = Files.getNameWithoutExtension(testCase.path.toString()); return dynamicTest(name, () -> { poll(packageName, testCase); }); }); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirLineDelimitedSourceTaskTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; public class SpoolDirLineDelimitedSourceTaskTest extends AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirJsonSourceTaskTest.class); @Override protected SpoolDirLineDelimitedSourceTask createTask() { return new SpoolDirLineDelimitedSourceTask(); } @Override protected Map settings() { Map settings = super.settings(); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF, "true"); settings.put(SpoolDirCsvSourceConnectorConfig.PARSER_TIMESTAMP_DATE_FORMATS_CONF, "yyyy-MM-dd'T'HH:mm:ss'Z'"); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_NULL_FIELD_INDICATOR_CONF, "BOTH"); return settings; } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirSchemaLessJsonSourceTaskTest.java ================================================ package com.github.jcustenborder.kafka.connect.spooldir; import com.google.common.io.Files; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicTest.dynamicTest; public class SpoolDirSchemaLessJsonSourceTaskTest extends AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirJsonSourceTaskTest.class); @Override protected SpoolDirSchemaLessJsonSourceTask createTask() { return new SpoolDirSchemaLessJsonSourceTask(); } @Override protected Map settings() { Map settings = super.settings(); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF, "true"); settings.put(SpoolDirCsvSourceConnectorConfig.PARSER_TIMESTAMP_DATE_FORMATS_CONF, "yyyy-MM-dd'T'HH:mm:ss'Z'"); settings.put(SpoolDirCsvSourceConnectorConfig.CSV_NULL_FIELD_INDICATOR_CONF, "BOTH"); return settings; } @TestFactory public Stream poll() throws IOException { final String packageName = "schemalessjson"; List testCases = loadTestCases(packageName); return testCases.stream().map(testCase -> { String name = Files.getNameWithoutExtension(testCase.path.toString()); return dynamicTest(name, () -> { poll(packageName, testCase); }); }); } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/TestCase.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.source.SourceRecord; import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; public class TestCase implements NamedTest { @JsonIgnore public Path path; public Map settings = new LinkedHashMap<>(); public Map offset = new LinkedHashMap<>(); public Schema keySchema; public Schema valueSchema; public List expected; @Override public void path(Path path) { this.path = path; } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/TestDataUtils.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir; import com.github.jcustenborder.kafka.connect.utils.jackson.ObjectMapperFactory; import com.google.common.base.Preconditions; import org.junit.jupiter.api.Test; import org.reflections.Reflections; import org.reflections.scanners.ResourcesScanner; import org.reflections.util.FilterBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Set; public class TestDataUtils { private static final Logger log = LoggerFactory.getLogger(TestDataUtils.class); @Test public void metadata() { log.info(Metadata.HEADER_DOCS); } public static List loadJsonResourceFiles(String packageName, Class cls) throws IOException { Preconditions.checkNotNull(packageName, "packageName cannot be null"); log.info("packageName = {}", packageName); // Preconditions.checkState(packageName.startsWith("/"), "packageName must start with a /."); Reflections reflections = new Reflections(packageName, new ResourcesScanner()); Set resources = reflections.getResources(new FilterBuilder.Include("^.*\\.json$")); List datas = new ArrayList(resources.size()); Path packagePath = Paths.get("/" + packageName.replace(".", "/")); for (String resource : resources) { log.trace("Loading resource {}", resource); Path resourcePath = Paths.get("/" + resource); Path relativePath = packagePath.relativize(resourcePath); File resourceFile = new File("/" + resource); T data; try (InputStream inputStream = cls.getResourceAsStream(resourceFile.getAbsolutePath())) { data = ObjectMapperFactory.INSTANCE.readValue(inputStream, cls); } catch (IOException ex) { if (log.isErrorEnabled()) { log.error("Exception thrown while loading {}", resourcePath, ex); } throw ex; } if (null != relativePath.getParent()) { data.path(relativePath); } else { data.path(relativePath); } datas.add(data); } return datas; } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/elf/SchemaConversionBuilderTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf; import com.github.jcustenborder.parsers.elf.ElfParser; import com.github.jcustenborder.parsers.elf.LogEntry; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.data.Struct; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; import java.time.LocalDate; import java.time.LocalTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.DynamicTest.dynamicTest; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class SchemaConversionBuilderTest { @TestFactory public Stream normalizeFieldName() { Map tests = new LinkedHashMap<>(); tests.put("date", "date"); tests.put("time", "time"); tests.put("x-edge-location", "x_edge_location"); tests.put("sc-bytes", "sc_bytes"); tests.put("c-ip", "c_ip"); tests.put("cs-method", "cs_method"); tests.put("cs(Host)", "cs_host"); tests.put("cs-uri-stem", "cs_uri_stem"); tests.put("sc-status", "sc_status"); tests.put("cs(Referer)", "cs_referer"); tests.put("cs(User-Agent)", "cs_user_agent"); tests.put("cs-uri-query", "cs_uri_query"); tests.put("cs(Cookie)", "cs_cookie"); tests.put("x-edge-result-type", "x_edge_result_type"); tests.put("x-edge-request-id", "x_edge_request_id"); tests.put("x-host-header", "x_host_header"); tests.put("cs-protocol", "cs_protocol"); tests.put("cs-bytes", "cs_bytes"); tests.put("time-taken", "time_taken"); return tests.entrySet().stream().map(e -> dynamicTest(e.getKey(), () -> { final String actual = SchemaConversionBuilder.normalizeFieldName(e.getKey()); assertEquals(e.getValue(), actual, "field name does not match."); })); } @Test public void foo() { ElfParser parser = mock(ElfParser.class); final Map> fieldTypes = ImmutableMap.of( "date", LocalDate.class, "time", LocalTime.class, "sc-bytes", Long.class, "sc-status", Integer.class ); final Map fieldData = ImmutableMap.of( "date", LocalDate.of(2011, 3, 14), "time", LocalTime.of(12, 0, 0), "sc-bytes", 12341L, "sc-status", 200 ); when(parser.fieldTypes()).thenReturn(fieldTypes); SchemaConversionBuilder schemaGenerator = new SchemaConversionBuilder(parser); SchemaConversion conversion = schemaGenerator.build(); assertNotNull(conversion, "conversion should not be null."); LogEntry entry = mock(LogEntry.class); when(entry.fieldTypes()).thenReturn(fieldTypes); when(entry.fieldData()).thenReturn(fieldData); SchemaAndValue actual = conversion.convert(entry); assertNotNull(actual, "actual should not be null"); // assertNotNull(actual.getKey(), "actual.getKey() should not be null"); assertNotNull(actual.schema(), "actual.getValue() should not be null"); assertNotNull(actual.value(), "actual.getValue() should not be null"); // actual.getValue()..validate(); //date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken } } ================================================ FILE: src/test/java/com/github/jcustenborder/kafka/connect/spooldir/elf/SpoolDirELFSourceTaskTest.java ================================================ /** * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.jcustenborder.kafka.connect.spooldir.elf; import com.github.jcustenborder.kafka.connect.spooldir.AbstractSpoolDirSourceTaskTest; import com.github.jcustenborder.kafka.connect.spooldir.TestCase; import com.google.common.io.Files; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicTest.dynamicTest; public class SpoolDirELFSourceTaskTest extends AbstractSpoolDirSourceTaskTest { private static final Logger log = LoggerFactory.getLogger(SpoolDirELFSourceTaskTest.class); @Override protected SpoolDirELFSourceTask createTask() { return new SpoolDirELFSourceTask(); } @Override protected Map settings() { Map settings = super.settings(); // settings.put(SpoolDirELFSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF, "true"); // settings.put(SpoolDirCsvSourceConnectorConfig.CSV_NULL_FIELD_INDICATOR_CONF, "BOTH"); // settings.put(SpoolDirCsvSourceConnectorConfig.PARSER_TIMESTAMP_DATE_FORMATS_CONF, "yyyy-MM-dd'T'HH:mm:ss'Z'"); return settings; } @TestFactory public Stream poll() throws IOException { final String packageName = "elf"; List testCases = loadTestCases(packageName); return testCases.stream().map(testCase -> { String name = Files.getNameWithoutExtension(testCase.path.toString()); return dynamicTest(name, () -> { poll(packageName, testCase); }); }); } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolBinaryFileSourceConnector/binary.json ================================================ { "name": "Binary File", "description": "This file will read the entire file and write it to Kafka as a binary file.", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^users\\d+\\.bin$", "topic": "users" } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolBinaryFileSourceConnector/fromXML.json ================================================ { "name": "Transform XML Files", "description": "This example will use the FromXml transformation to read the binary data based on the supplied XSD. This allows files to be converted to strongly typed data based on the XSD. Once Kafka connect has converted the data it can be stored as AVRO, JSON, or whatever converter the user chooses.", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^users\\d+\\.bin$", "topic": "users", "transforms": "FromXml", "transforms.FromXml.type":"com.github.jcustenborder.kafka.connect.transform.xml.FromXml$Value", "transforms.FromXml.schema.path": "file:///books.xsd" } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirBinaryFileSourceConnector/binary.json ================================================ { "name": "Binary File", "description": "This file will read the entire file and write it to Kafka as a binary file.", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^users\\d+\\.bin$", "topic": "users" } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirBinaryFileSourceConnector/fromXML.json ================================================ { "name": "Transform XML Files", "description": "This example will use the FromXml transformation to read the binary data based on the supplied XSD. This allows files to be converted to strongly typed data based on the XSD. Once Kafka connect has converted the data it can be stored as AVRO, JSON, or whatever converter the user chooses.", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^users\\d+\\.bin$", "topic": "users", "transforms": "FromXml", "transforms.FromXml.type":"com.github.jcustenborder.kafka.connect.transform.xml.FromXml$Value", "transforms.FromXml.schema.path": "file:///books.xsd" } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector/schema.json ================================================ { "name": "CSV with schema", "description": "This example will read csv files and write them to Kafka parsing them to the\nschema specified in ``key.schema`` and ``value.schema``.", "note": "The data for the following example is formatted as such.\nid,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color\n1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^users\\d+\\.csv", "topic": "users", "key.schema": "{\n \"name\" : \"com.example.users.UserKey\",\n \"type\" : \"STRUCT\",\n \"isOptional\" : false,\n \"fieldSchemas\" : {\n \"id\" : {\n \"type\" : \"INT64\",\n \"isOptional\" : false\n }\n }\n}\n", "value.schema": "{\n \"name\" : \"com.example.users.User\",\n \"type\" : \"STRUCT\",\n \"isOptional\" : false,\n \"fieldSchemas\" : {\n \"id\" : {\n \"type\" : \"INT64\",\n \"isOptional\" : false\n },\n \"first_name\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"last_name\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"email\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"gender\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"ip_address\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"last_login\" : {\n \"name\" : \"org.apache.kafka.connect.data.Timestamp\",\n \"type\" : \"INT64\",\n \"version\" : 1,\n \"isOptional\" : false\n },\n \"account_balance\" : {\n \"name\" : \"org.apache.kafka.connect.data.Decimal\",\n \"type\" : \"BYTES\",\n \"version\" : 1,\n \"parameters\" : {\n \"scale\" : \"2\"\n },\n \"isOptional\" : true\n },\n \"country\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"favorite_color\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n }\n }\n}\n" } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector/schemaheaders.json ================================================ { "name": "CSV with Headers as fields", "description": "This example will use a transformation to copy data from the header(s) of the message to field(s) in the message.", "note": "The data for the following example is formatted as such.\nid,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color\n1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^users\\d+\\.csv", "topic": "users", "key.schema": "{\n \"name\" : \"com.example.users.UserKey\",\n \"type\" : \"STRUCT\",\n \"isOptional\" : false,\n \"fieldSchemas\" : {\n \"id\" : {\n \"type\" : \"INT64\",\n \"isOptional\" : false\n }\n }\n}\n", "value.schema": "{\n \"name\" : \"com.example.users.User\",\n \"type\" : \"STRUCT\",\n \"isOptional\" : false,\n \"fieldSchemas\" : {\n \"id\" : {\n \"type\" : \"INT64\",\n \"isOptional\" : false\n },\n \"first_name\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"last_name\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"email\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"gender\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"ip_address\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"last_login\" : {\n \"name\" : \"org.apache.kafka.connect.data.Timestamp\",\n \"type\" : \"INT64\",\n \"version\" : 1,\n \"isOptional\" : false\n },\n \"account_balance\" : {\n \"name\" : \"org.apache.kafka.connect.data.Decimal\",\n \"type\" : \"BYTES\",\n \"version\" : 1,\n \"parameters\" : {\n \"scale\" : \"2\"\n },\n \"isOptional\" : true\n },\n \"country\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n },\n \"favorite_color\" : {\n \"type\" : \"STRING\",\n \"isOptional\" : true\n }\n }\n}\n" }, "transformations": { "headerToField": { "type": "com.github.jcustenborder.kafka.connect.transform.common.HeaderToField$Value", "header.mappings" : "file.path:STRING:file_path,file.name:STRING:file_name,file.last.modified:INT64(Timestamp):file_last_modified" } } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector/tsv.json ================================================ { "name": "TSV input file", "description": "This example will read a tab separated file. This method is very similar to reading a standard CSV file.", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^users\\d+\\.tsv", "topic": "users", "csv.separator.char": 11 } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnector/test.json ================================================ { "name":"Json", "description":"This example will read json from the input directory.", "config":{ "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern":"^users\\d+\\.json$", "topic":"users" } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirLineDelimitedSourceConnector/fix.json ================================================ { "description" : "This example will read files in a directory line by line and parse them using kafka-connect-transform-fix to a FIX representation of the data.", "name" : "FIX encoded lines", "config" : { "topic" : "fix", "input.path" : "/tmp", "input.file.pattern" : "^.+\\.fix$", "error.path" : "/tmp", "finished.path" : "/tmp" }, "transformations" : { "fromFix" : { "type" : "com.github.jcustenborder.kafka.connect.transform.fix.FromFIX$Value" } }, "output" : { "sourcePartition" : { }, "sourceOffset" : { }, "topic" : "fix", "kafkaPartition" : 0, "valueSchema" : { "name" : "fix42.NewOrderSingle", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "Account" : { "type" : "STRING", "parameters" : { "fix.field" : "1" }, "isOptional" : true }, "CashOrderQty" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "152" }, "isOptional" : true }, "CheckSum" : { "type" : "STRING", "parameters" : { "fix.field" : "10" }, "isOptional" : true }, "ClOrdID" : { "type" : "STRING", "parameters" : { "fix.field" : "11" }, "isOptional" : true }, "ClearingAccount" : { "type" : "STRING", "parameters" : { "fix.field" : "440" }, "isOptional" : true }, "ClearingFirm" : { "type" : "STRING", "parameters" : { "fix.field" : "439" }, "isOptional" : true }, "ClientID" : { "type" : "STRING", "parameters" : { "fix.field" : "109" }, "isOptional" : true }, "CommType" : { "type" : "STRING", "parameters" : { "fix.field" : "13" }, "isOptional" : true }, "Commission" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "12" }, "isOptional" : true }, "ComplianceID" : { "type" : "STRING", "parameters" : { "fix.field" : "376" }, "isOptional" : true }, "ContractMultiplier" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "231" }, "isOptional" : true }, "CouponRate" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "223" }, "isOptional" : true }, "CoveredOrUncovered" : { "type" : "INT32", "parameters" : { "fix.field" : "203" }, "isOptional" : true }, "Currency" : { "type" : "STRING", "parameters" : { "fix.field" : "15" }, "isOptional" : true }, "CustomerOrFirm" : { "type" : "INT32", "parameters" : { "fix.field" : "204" }, "isOptional" : true }, "DiscretionInst" : { "type" : "STRING", "parameters" : { "fix.field" : "388" }, "isOptional" : true }, "DiscretionOffset" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "389" }, "isOptional" : true }, "EffectiveTime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "168" }, "isOptional" : true }, "EncodedIssuer" : { "type" : "STRING", "parameters" : { "fix.field" : "349" }, "isOptional" : true }, "EncodedIssuerLen" : { "type" : "INT32", "parameters" : { "fix.field" : "348" }, "isOptional" : true }, "EncodedSecurityDesc" : { "type" : "STRING", "parameters" : { "fix.field" : "351" }, "isOptional" : true }, "EncodedSecurityDescLen" : { "type" : "INT32", "parameters" : { "fix.field" : "350" }, "isOptional" : true }, "EncodedText" : { "type" : "STRING", "parameters" : { "fix.field" : "355" }, "isOptional" : true }, "EncodedTextLen" : { "type" : "INT32", "parameters" : { "fix.field" : "354" }, "isOptional" : true }, "ExDestination" : { "type" : "STRING", "parameters" : { "fix.field" : "100" }, "isOptional" : true }, "ExecBroker" : { "type" : "STRING", "parameters" : { "fix.field" : "76" }, "isOptional" : true }, "ExecInst" : { "type" : "STRING", "parameters" : { "fix.field" : "18" }, "isOptional" : true }, "ExpireDate" : { "type" : "STRING", "parameters" : { "fix.field" : "432" }, "isOptional" : true }, "ExpireTime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "126" }, "isOptional" : true }, "ForexReq" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "121" }, "isOptional" : true }, "FutSettDate" : { "type" : "STRING", "parameters" : { "fix.field" : "64" }, "isOptional" : true }, "FutSettDate2" : { "type" : "STRING", "parameters" : { "fix.field" : "193" }, "isOptional" : true }, "GTBookingInst" : { "type" : "INT32", "parameters" : { "fix.field" : "427" }, "isOptional" : true }, "HandlInst" : { "type" : "STRING", "parameters" : { "fix.field" : "21" }, "isOptional" : true }, "IDSource" : { "type" : "STRING", "parameters" : { "fix.field" : "22" }, "isOptional" : true }, "IOIID" : { "type" : "STRING", "parameters" : { "fix.field" : "23" }, "isOptional" : true }, "Issuer" : { "type" : "STRING", "parameters" : { "fix.field" : "106" }, "isOptional" : true }, "LocateReqd" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "114" }, "isOptional" : true }, "MaturityDay" : { "type" : "STRING", "parameters" : { "fix.field" : "205" }, "isOptional" : true }, "MaturityMonthYear" : { "type" : "STRING", "parameters" : { "fix.field" : "200" }, "isOptional" : true }, "MaxFloor" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "111" }, "isOptional" : true }, "MaxShow" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "210" }, "isOptional" : true }, "MinQty" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "110" }, "isOptional" : true }, "NoAllocs" : { "type" : "INT32", "parameters" : { "fix.field" : "78" }, "isOptional" : true }, "NoTradingSessions" : { "type" : "INT32", "parameters" : { "fix.field" : "386" }, "isOptional" : true }, "OpenClose" : { "type" : "STRING", "parameters" : { "fix.field" : "77" }, "isOptional" : true }, "OptAttribute" : { "type" : "STRING", "parameters" : { "fix.field" : "206" }, "isOptional" : true }, "OrdType" : { "type" : "STRING", "parameters" : { "fix.field" : "40" }, "isOptional" : true }, "OrderQty" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "38" }, "isOptional" : true }, "OrderQty2" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "192" }, "isOptional" : true }, "PegDifference" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "211" }, "isOptional" : true }, "PrevClosePx" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "140" }, "isOptional" : true }, "Price" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "44" }, "isOptional" : true }, "ProcessCode" : { "type" : "STRING", "parameters" : { "fix.field" : "81" }, "isOptional" : true }, "PutOrCall" : { "type" : "INT32", "parameters" : { "fix.field" : "201" }, "isOptional" : true }, "QuoteID" : { "type" : "STRING", "parameters" : { "fix.field" : "117" }, "isOptional" : true }, "Rule80A" : { "type" : "STRING", "parameters" : { "fix.field" : "47" }, "isOptional" : true }, "SecurityDesc" : { "type" : "STRING", "parameters" : { "fix.field" : "107" }, "isOptional" : true }, "SecurityExchange" : { "type" : "STRING", "parameters" : { "fix.field" : "207" }, "isOptional" : true }, "SecurityID" : { "type" : "STRING", "parameters" : { "fix.field" : "48" }, "isOptional" : true }, "SecurityType" : { "type" : "STRING", "parameters" : { "fix.field" : "167" }, "isOptional" : true }, "SettlCurrency" : { "type" : "STRING", "parameters" : { "fix.field" : "120" }, "isOptional" : true }, "SettlmntTyp" : { "type" : "STRING", "parameters" : { "fix.field" : "63" }, "isOptional" : true }, "Side" : { "type" : "STRING", "parameters" : { "fix.field" : "54" }, "isOptional" : true }, "Signature" : { "type" : "STRING", "parameters" : { "fix.field" : "89" }, "isOptional" : true }, "SignatureLength" : { "type" : "INT32", "parameters" : { "fix.field" : "93" }, "isOptional" : true }, "SolicitedFlag" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "377" }, "isOptional" : true }, "StopPx" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "99" }, "isOptional" : true }, "StrikePrice" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "202" }, "isOptional" : true }, "Symbol" : { "type" : "STRING", "parameters" : { "fix.field" : "55" }, "isOptional" : true }, "SymbolSfx" : { "type" : "STRING", "parameters" : { "fix.field" : "65" }, "isOptional" : true }, "Text" : { "type" : "STRING", "parameters" : { "fix.field" : "58" }, "isOptional" : true }, "TimeInForce" : { "type" : "STRING", "parameters" : { "fix.field" : "59" }, "isOptional" : true }, "TransactTime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "60" }, "isOptional" : true } } }, "value" : { "schema" : { "name" : "fix42.NewOrderSingle", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "Account" : { "type" : "STRING", "parameters" : { "fix.field" : "1" }, "isOptional" : true }, "CashOrderQty" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "152" }, "isOptional" : true }, "CheckSum" : { "type" : "STRING", "parameters" : { "fix.field" : "10" }, "isOptional" : true }, "ClOrdID" : { "type" : "STRING", "parameters" : { "fix.field" : "11" }, "isOptional" : true }, "ClearingAccount" : { "type" : "STRING", "parameters" : { "fix.field" : "440" }, "isOptional" : true }, "ClearingFirm" : { "type" : "STRING", "parameters" : { "fix.field" : "439" }, "isOptional" : true }, "ClientID" : { "type" : "STRING", "parameters" : { "fix.field" : "109" }, "isOptional" : true }, "CommType" : { "type" : "STRING", "parameters" : { "fix.field" : "13" }, "isOptional" : true }, "Commission" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "12" }, "isOptional" : true }, "ComplianceID" : { "type" : "STRING", "parameters" : { "fix.field" : "376" }, "isOptional" : true }, "ContractMultiplier" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "231" }, "isOptional" : true }, "CouponRate" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "223" }, "isOptional" : true }, "CoveredOrUncovered" : { "type" : "INT32", "parameters" : { "fix.field" : "203" }, "isOptional" : true }, "Currency" : { "type" : "STRING", "parameters" : { "fix.field" : "15" }, "isOptional" : true }, "CustomerOrFirm" : { "type" : "INT32", "parameters" : { "fix.field" : "204" }, "isOptional" : true }, "DiscretionInst" : { "type" : "STRING", "parameters" : { "fix.field" : "388" }, "isOptional" : true }, "DiscretionOffset" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "389" }, "isOptional" : true }, "EffectiveTime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "168" }, "isOptional" : true }, "EncodedIssuer" : { "type" : "STRING", "parameters" : { "fix.field" : "349" }, "isOptional" : true }, "EncodedIssuerLen" : { "type" : "INT32", "parameters" : { "fix.field" : "348" }, "isOptional" : true }, "EncodedSecurityDesc" : { "type" : "STRING", "parameters" : { "fix.field" : "351" }, "isOptional" : true }, "EncodedSecurityDescLen" : { "type" : "INT32", "parameters" : { "fix.field" : "350" }, "isOptional" : true }, "EncodedText" : { "type" : "STRING", "parameters" : { "fix.field" : "355" }, "isOptional" : true }, "EncodedTextLen" : { "type" : "INT32", "parameters" : { "fix.field" : "354" }, "isOptional" : true }, "ExDestination" : { "type" : "STRING", "parameters" : { "fix.field" : "100" }, "isOptional" : true }, "ExecBroker" : { "type" : "STRING", "parameters" : { "fix.field" : "76" }, "isOptional" : true }, "ExecInst" : { "type" : "STRING", "parameters" : { "fix.field" : "18" }, "isOptional" : true }, "ExpireDate" : { "type" : "STRING", "parameters" : { "fix.field" : "432" }, "isOptional" : true }, "ExpireTime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "126" }, "isOptional" : true }, "ForexReq" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "121" }, "isOptional" : true }, "FutSettDate" : { "type" : "STRING", "parameters" : { "fix.field" : "64" }, "isOptional" : true }, "FutSettDate2" : { "type" : "STRING", "parameters" : { "fix.field" : "193" }, "isOptional" : true }, "GTBookingInst" : { "type" : "INT32", "parameters" : { "fix.field" : "427" }, "isOptional" : true }, "HandlInst" : { "type" : "STRING", "parameters" : { "fix.field" : "21" }, "isOptional" : true }, "IDSource" : { "type" : "STRING", "parameters" : { "fix.field" : "22" }, "isOptional" : true }, "IOIID" : { "type" : "STRING", "parameters" : { "fix.field" : "23" }, "isOptional" : true }, "Issuer" : { "type" : "STRING", "parameters" : { "fix.field" : "106" }, "isOptional" : true }, "LocateReqd" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "114" }, "isOptional" : true }, "MaturityDay" : { "type" : "STRING", "parameters" : { "fix.field" : "205" }, "isOptional" : true }, "MaturityMonthYear" : { "type" : "STRING", "parameters" : { "fix.field" : "200" }, "isOptional" : true }, "MaxFloor" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "111" }, "isOptional" : true }, "MaxShow" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "210" }, "isOptional" : true }, "MinQty" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "110" }, "isOptional" : true }, "NoAllocs" : { "type" : "INT32", "parameters" : { "fix.field" : "78" }, "isOptional" : true }, "NoTradingSessions" : { "type" : "INT32", "parameters" : { "fix.field" : "386" }, "isOptional" : true }, "OpenClose" : { "type" : "STRING", "parameters" : { "fix.field" : "77" }, "isOptional" : true }, "OptAttribute" : { "type" : "STRING", "parameters" : { "fix.field" : "206" }, "isOptional" : true }, "OrdType" : { "type" : "STRING", "parameters" : { "fix.field" : "40" }, "isOptional" : true }, "OrderQty" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "38" }, "isOptional" : true }, "OrderQty2" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "192" }, "isOptional" : true }, "PegDifference" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "211" }, "isOptional" : true }, "PrevClosePx" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "140" }, "isOptional" : true }, "Price" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "44" }, "isOptional" : true }, "ProcessCode" : { "type" : "STRING", "parameters" : { "fix.field" : "81" }, "isOptional" : true }, "PutOrCall" : { "type" : "INT32", "parameters" : { "fix.field" : "201" }, "isOptional" : true }, "QuoteID" : { "type" : "STRING", "parameters" : { "fix.field" : "117" }, "isOptional" : true }, "Rule80A" : { "type" : "STRING", "parameters" : { "fix.field" : "47" }, "isOptional" : true }, "SecurityDesc" : { "type" : "STRING", "parameters" : { "fix.field" : "107" }, "isOptional" : true }, "SecurityExchange" : { "type" : "STRING", "parameters" : { "fix.field" : "207" }, "isOptional" : true }, "SecurityID" : { "type" : "STRING", "parameters" : { "fix.field" : "48" }, "isOptional" : true }, "SecurityType" : { "type" : "STRING", "parameters" : { "fix.field" : "167" }, "isOptional" : true }, "SettlCurrency" : { "type" : "STRING", "parameters" : { "fix.field" : "120" }, "isOptional" : true }, "SettlmntTyp" : { "type" : "STRING", "parameters" : { "fix.field" : "63" }, "isOptional" : true }, "Side" : { "type" : "STRING", "parameters" : { "fix.field" : "54" }, "isOptional" : true }, "Signature" : { "type" : "STRING", "parameters" : { "fix.field" : "89" }, "isOptional" : true }, "SignatureLength" : { "type" : "INT32", "parameters" : { "fix.field" : "93" }, "isOptional" : true }, "SolicitedFlag" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "377" }, "isOptional" : true }, "StopPx" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "99" }, "isOptional" : true }, "StrikePrice" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "202" }, "isOptional" : true }, "Symbol" : { "type" : "STRING", "parameters" : { "fix.field" : "55" }, "isOptional" : true }, "SymbolSfx" : { "type" : "STRING", "parameters" : { "fix.field" : "65" }, "isOptional" : true }, "Text" : { "type" : "STRING", "parameters" : { "fix.field" : "58" }, "isOptional" : true }, "TimeInForce" : { "type" : "STRING", "parameters" : { "fix.field" : "59" }, "isOptional" : true }, "TransactTime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "60" }, "isOptional" : true } } }, "fieldValues" : [ { "name" : "Account", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "1" }, "isOptional" : true } }, { "name" : "CashOrderQty", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "152" }, "isOptional" : true } }, { "name" : "CheckSum", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "10" }, "isOptional" : true } }, { "name" : "ClOrdID", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "11" }, "isOptional" : true }, "storage" : "NF 0542/03232009" }, { "name" : "ClearingAccount", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "440" }, "isOptional" : true } }, { "name" : "ClearingFirm", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "439" }, "isOptional" : true } }, { "name" : "ClientID", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "109" }, "isOptional" : true } }, { "name" : "CommType", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "13" }, "isOptional" : true } }, { "name" : "Commission", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "12" }, "isOptional" : true } }, { "name" : "ComplianceID", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "376" }, "isOptional" : true } }, { "name" : "ContractMultiplier", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "231" }, "isOptional" : true } }, { "name" : "CouponRate", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "223" }, "isOptional" : true } }, { "name" : "CoveredOrUncovered", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "203" }, "isOptional" : true } }, { "name" : "Currency", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "15" }, "isOptional" : true } }, { "name" : "CustomerOrFirm", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "204" }, "isOptional" : true } }, { "name" : "DiscretionInst", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "388" }, "isOptional" : true } }, { "name" : "DiscretionOffset", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "389" }, "isOptional" : true } }, { "name" : "EffectiveTime", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "168" }, "isOptional" : true } }, { "name" : "EncodedIssuer", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "349" }, "isOptional" : true } }, { "name" : "EncodedIssuerLen", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "348" }, "isOptional" : true } }, { "name" : "EncodedSecurityDesc", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "351" }, "isOptional" : true } }, { "name" : "EncodedSecurityDescLen", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "350" }, "isOptional" : true } }, { "name" : "EncodedText", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "355" }, "isOptional" : true } }, { "name" : "EncodedTextLen", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "354" }, "isOptional" : true } }, { "name" : "ExDestination", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "100" }, "isOptional" : true } }, { "name" : "ExecBroker", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "76" }, "isOptional" : true } }, { "name" : "ExecInst", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "18" }, "isOptional" : true } }, { "name" : "ExpireDate", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "432" }, "isOptional" : true } }, { "name" : "ExpireTime", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "126" }, "isOptional" : true } }, { "name" : "ForexReq", "schema" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "121" }, "isOptional" : true } }, { "name" : "FutSettDate", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "64" }, "isOptional" : true } }, { "name" : "FutSettDate2", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "193" }, "isOptional" : true } }, { "name" : "GTBookingInst", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "427" }, "isOptional" : true } }, { "name" : "HandlInst", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "21" }, "isOptional" : true }, "storage" : "1" }, { "name" : "IDSource", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "22" }, "isOptional" : true } }, { "name" : "IOIID", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "23" }, "isOptional" : true } }, { "name" : "Issuer", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "106" }, "isOptional" : true } }, { "name" : "LocateReqd", "schema" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "114" }, "isOptional" : true } }, { "name" : "MaturityDay", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "205" }, "isOptional" : true } }, { "name" : "MaturityMonthYear", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "200" }, "isOptional" : true } }, { "name" : "MaxFloor", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "111" }, "isOptional" : true } }, { "name" : "MaxShow", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "210" }, "isOptional" : true } }, { "name" : "MinQty", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "110" }, "isOptional" : true } }, { "name" : "NoAllocs", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "78" }, "isOptional" : true } }, { "name" : "NoTradingSessions", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "386" }, "isOptional" : true } }, { "name" : "OpenClose", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "77" }, "isOptional" : true } }, { "name" : "OptAttribute", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "206" }, "isOptional" : true } }, { "name" : "OrdType", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "40" }, "isOptional" : true }, "storage" : "1" }, { "name" : "OrderQty", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "38" }, "isOptional" : true }, "storage" : 100.0 }, { "name" : "OrderQty2", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "192" }, "isOptional" : true } }, { "name" : "PegDifference", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "211" }, "isOptional" : true } }, { "name" : "PrevClosePx", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "140" }, "isOptional" : true } }, { "name" : "Price", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "44" }, "isOptional" : true } }, { "name" : "ProcessCode", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "81" }, "isOptional" : true } }, { "name" : "PutOrCall", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "201" }, "isOptional" : true } }, { "name" : "QuoteID", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "117" }, "isOptional" : true } }, { "name" : "Rule80A", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "47" }, "isOptional" : true }, "storage" : "A" }, { "name" : "SecurityDesc", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "107" }, "isOptional" : true } }, { "name" : "SecurityExchange", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "207" }, "isOptional" : true }, "storage" : "N" }, { "name" : "SecurityID", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "48" }, "isOptional" : true } }, { "name" : "SecurityType", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "167" }, "isOptional" : true } }, { "name" : "SettlCurrency", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "120" }, "isOptional" : true } }, { "name" : "SettlmntTyp", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "63" }, "isOptional" : true } }, { "name" : "Side", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "54" }, "isOptional" : true }, "storage" : "1" }, { "name" : "Signature", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "89" }, "isOptional" : true } }, { "name" : "SignatureLength", "schema" : { "type" : "INT32", "parameters" : { "fix.field" : "93" }, "isOptional" : true } }, { "name" : "SolicitedFlag", "schema" : { "type" : "BOOLEAN", "parameters" : { "fix.field" : "377" }, "isOptional" : true } }, { "name" : "StopPx", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "99" }, "isOptional" : true } }, { "name" : "StrikePrice", "schema" : { "type" : "FLOAT64", "parameters" : { "fix.field" : "202" }, "isOptional" : true } }, { "name" : "Symbol", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "55" }, "isOptional" : true }, "storage" : "CVS" }, { "name" : "SymbolSfx", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "65" }, "isOptional" : true } }, { "name" : "Text", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "58" }, "isOptional" : true } }, { "name" : "TimeInForce", "schema" : { "type" : "STRING", "parameters" : { "fix.field" : "59" }, "isOptional" : true }, "storage" : "0" }, { "name" : "TransactTime", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "fix.field" : "60" }, "isOptional" : true }, "storage" : 1237822829000 } ] }, "headers" : [ ] } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/avro/FieldsMatch.json ================================================ { "settings" : { "avro.first.row.as.header" : "true" }, "offset" : { }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 0 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 0 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 64319.61 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 82687.61 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 77124.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 55799.93 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 40385.72 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 48183.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 52924.60 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -16688.36 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 67643.16 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 79170.84 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.avro" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 36684.92 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.avro" }, { "name" : "file.name.without.extension", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1611115121380-0/input/FieldsMatch.avro" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2291 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1611115121951 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/binary/DataHasMoreFields.data ================================================ asdifoasodfasdfargasdfasdfasdgfrasdfasdfasdfa ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/binary/DataHasMoreFields.json ================================================ { "settings": {}, "offset": {}, "expected": [ { "sourcePartition": { "fileName": "DataHasMoreFields.binary" }, "sourceOffset": { "offset": 0 }, "topic": "testing", "valueSchema": { "type": "BYTES", "isOptional": false }, "value": "YXNkaWZvYXNvZGZhc2RmYXJnYXNkZmFzZGZhc2RnZnJhc2RmYXNkZmFzZGZh", "headers": [ { "name": "file.name", "schema": { "type": "STRING", "isOptional": false }, "storage": "DataHasMoreFields.binary" }, { "name": "file.path", "schema": { "type": "STRING", "isOptional": false }, "storage": "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.binary" }, { "name": "file.length", "schema": { "type": "INT64", "isOptional": false }, "storage": 5153 }, { "name": "file.offset", "schema": { "type": "INT64", "isOptional": false }, "storage": 0 }, { "name": "file.last.modified", "schema": { "name": "org.apache.kafka.connect.data.Timestamp", "type": "INT64", "version": 1, "isOptional": false }, "storage": 1559925239000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.data ================================================ id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN, 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "BlankLines.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "BlankLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/DataHasMoreFields.data ================================================ id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color,column11 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313,asdsa 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf,asd 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN,,f 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414,as 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN,,g 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR,,g 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d,h 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU,,j 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN,,f 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9,s 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36,dfg 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909,hg 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN,,s 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943,fgs 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119,sdfgs 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH,,sdfg 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID,,sdfg 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN,,sdfg 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL,,dsf 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128,sdfgsdf ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/DataHasMoreFields.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/DataHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2050 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.data ================================================ id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN, 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FileModeFieldFieldsMatch.data ================================================ id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,2015-11-14T10:34:09Z,819.47,CN, 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FileModeFieldFieldsMatch.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/FileModeFieldFieldsMatch.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1988 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SchemaHasMoreFields.data ================================================ id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SchemaHasMoreFields.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SchemaHasMoreFields.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1863 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SourceOffset.data ================================================ id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN, 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SourceOffset.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { "offset" : 10 }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/SourceOffset.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1968 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.data ================================================ #skip this line id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN, 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.json ================================================ { "settings" : { "csv.first.row.as.header" : "true", "csv.skip.lines": "1", "schema.generation.enabled": "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithHeaderSkipLines.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithHeaderSkipLines.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithoutHeader.data ================================================ 1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf 3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN, 4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, 6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, 7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d 8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, 9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, 10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, 14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, 17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, 18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, 19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, 20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithoutHeader.json ================================================ { "settings" : { "csv.first.row.as.header" : "false" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] }, { "sourcePartition" : { "fileName" : "WithoutHeader.csv" }, "sourceOffset" : { "offset" : 20 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "WithoutHeader.csv" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithoutHeader.csv" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1870 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925244000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/elf/SpoolDirELFSourceConnector/example.json ================================================ { "name": "Standard", "description": "This example will read Extended Log Format files and write them to Kafka.", "config": { "finished.path": "/tmp", "input.path": "/tmp", "error.path": "/tmp", "input.file.pattern": "^server\\d+\\.log", "topic": "users" } } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/elf/elf/FieldsMatch.data ================================================ #Software: xyz 1.2.3.4 #Version: 1.0 #Start-Date: 2019-10-23 22:00:00 #Date: 2019-10-23 22:00:00 #Fields: date time time-taken ip1 user gp id1 res cat1 status act method ct scheme host port ext ua ip2 bytes1 bytes2 id2 err1 err2 err3 str cr cat2 #Remark: 1234567890 "xyz" "1.2.3.4" "type" 2019-10-23 21:00:00 1234 1.2.3.4 xyz abc - OK "cat" 0 zyx unk - pro site 123 - - 1.2.3.4 0 0 - none - - none - "cat" 2019-10-23 21:00:00 123 1.2.3.4 xyz abc - OK "dog" 0 zyx unk - pro site 123 - - 1.2.3.4 0 0 - none - - none - "dog" ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/elf/elf/FieldsMatch.json ================================================ { "settings" : { }, "offset" : { }, "expected" : [ { "sourcePartition" : { "fileName" : "FieldsMatch.elf" }, "sourceOffset" : { "offset" : 0 }, "topic" : "testing", "valueSchema" : { "name" : "com.github.jcustenborder.kafka.connect.spooldir.LogEntry", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "date" : { "name" : "org.apache.kafka.connect.data.Date", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "date" }, "isOptional" : true }, "time" : { "name" : "org.apache.kafka.connect.data.Time", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "time" }, "isOptional" : true }, "time_taken" : { "type" : "FLOAT64", "parameters" : { "logField" : "time-taken" }, "isOptional" : true }, "ip1" : { "type" : "STRING", "parameters" : { "logField" : "ip1" }, "isOptional" : true }, "user" : { "type" : "STRING", "parameters" : { "logField" : "user" }, "isOptional" : true }, "gp" : { "type" : "STRING", "parameters" : { "logField" : "gp" }, "isOptional" : true }, "id1" : { "type" : "STRING", "parameters" : { "logField" : "id1" }, "isOptional" : true }, "res" : { "type" : "STRING", "parameters" : { "logField" : "res" }, "isOptional" : true }, "cat1" : { "type" : "STRING", "parameters" : { "logField" : "cat1" }, "isOptional" : true }, "status" : { "type" : "STRING", "parameters" : { "logField" : "status" }, "isOptional" : true }, "act" : { "type" : "STRING", "parameters" : { "logField" : "act" }, "isOptional" : true }, "method" : { "type" : "STRING", "parameters" : { "logField" : "method" }, "isOptional" : true }, "ct" : { "type" : "STRING", "parameters" : { "logField" : "ct" }, "isOptional" : true }, "scheme" : { "type" : "STRING", "parameters" : { "logField" : "scheme" }, "isOptional" : true }, "host" : { "type" : "STRING", "parameters" : { "logField" : "host" }, "isOptional" : true }, "port" : { "type" : "STRING", "parameters" : { "logField" : "port" }, "isOptional" : true }, "ext" : { "type" : "STRING", "parameters" : { "logField" : "ext" }, "isOptional" : true }, "ua" : { "type" : "STRING", "parameters" : { "logField" : "ua" }, "isOptional" : true }, "ip2" : { "type" : "STRING", "parameters" : { "logField" : "ip2" }, "isOptional" : true }, "bytes1" : { "type" : "INT64", "parameters" : { "logField" : "bytes1" }, "isOptional" : true }, "bytes2" : { "type" : "INT64", "parameters" : { "logField" : "bytes2" }, "isOptional" : true }, "id2" : { "type" : "STRING", "parameters" : { "logField" : "id2" }, "isOptional" : true }, "err1" : { "type" : "STRING", "parameters" : { "logField" : "err1" }, "isOptional" : true }, "err2" : { "type" : "STRING", "parameters" : { "logField" : "err2" }, "isOptional" : true }, "err3" : { "type" : "STRING", "parameters" : { "logField" : "err3" }, "isOptional" : true }, "str" : { "type" : "STRING", "parameters" : { "logField" : "str" }, "isOptional" : true }, "cr" : { "type" : "STRING", "parameters" : { "logField" : "cr" }, "isOptional" : true }, "cat2" : { "type" : "STRING", "parameters" : { "logField" : "cat2" }, "isOptional" : true }, "datetime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "logField" : "date,time" }, "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.github.jcustenborder.kafka.connect.spooldir.LogEntry", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "date" : { "name" : "org.apache.kafka.connect.data.Date", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "date" }, "isOptional" : true }, "time" : { "name" : "org.apache.kafka.connect.data.Time", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "time" }, "isOptional" : true }, "time_taken" : { "type" : "FLOAT64", "parameters" : { "logField" : "time-taken" }, "isOptional" : true }, "ip1" : { "type" : "STRING", "parameters" : { "logField" : "ip1" }, "isOptional" : true }, "user" : { "type" : "STRING", "parameters" : { "logField" : "user" }, "isOptional" : true }, "gp" : { "type" : "STRING", "parameters" : { "logField" : "gp" }, "isOptional" : true }, "id1" : { "type" : "STRING", "parameters" : { "logField" : "id1" }, "isOptional" : true }, "res" : { "type" : "STRING", "parameters" : { "logField" : "res" }, "isOptional" : true }, "cat1" : { "type" : "STRING", "parameters" : { "logField" : "cat1" }, "isOptional" : true }, "status" : { "type" : "STRING", "parameters" : { "logField" : "status" }, "isOptional" : true }, "act" : { "type" : "STRING", "parameters" : { "logField" : "act" }, "isOptional" : true }, "method" : { "type" : "STRING", "parameters" : { "logField" : "method" }, "isOptional" : true }, "ct" : { "type" : "STRING", "parameters" : { "logField" : "ct" }, "isOptional" : true }, "scheme" : { "type" : "STRING", "parameters" : { "logField" : "scheme" }, "isOptional" : true }, "host" : { "type" : "STRING", "parameters" : { "logField" : "host" }, "isOptional" : true }, "port" : { "type" : "STRING", "parameters" : { "logField" : "port" }, "isOptional" : true }, "ext" : { "type" : "STRING", "parameters" : { "logField" : "ext" }, "isOptional" : true }, "ua" : { "type" : "STRING", "parameters" : { "logField" : "ua" }, "isOptional" : true }, "ip2" : { "type" : "STRING", "parameters" : { "logField" : "ip2" }, "isOptional" : true }, "bytes1" : { "type" : "INT64", "parameters" : { "logField" : "bytes1" }, "isOptional" : true }, "bytes2" : { "type" : "INT64", "parameters" : { "logField" : "bytes2" }, "isOptional" : true }, "id2" : { "type" : "STRING", "parameters" : { "logField" : "id2" }, "isOptional" : true }, "err1" : { "type" : "STRING", "parameters" : { "logField" : "err1" }, "isOptional" : true }, "err2" : { "type" : "STRING", "parameters" : { "logField" : "err2" }, "isOptional" : true }, "err3" : { "type" : "STRING", "parameters" : { "logField" : "err3" }, "isOptional" : true }, "str" : { "type" : "STRING", "parameters" : { "logField" : "str" }, "isOptional" : true }, "cr" : { "type" : "STRING", "parameters" : { "logField" : "cr" }, "isOptional" : true }, "cat2" : { "type" : "STRING", "parameters" : { "logField" : "cat2" }, "isOptional" : true }, "datetime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "logField" : "date,time" }, "isOptional" : true } } }, "fieldValues" : [ { "name" : "date", "schema" : { "name" : "org.apache.kafka.connect.data.Date", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "date" }, "isOptional" : true }, "storage" : 18192 }, { "name" : "time", "schema" : { "name" : "org.apache.kafka.connect.data.Time", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "time" }, "isOptional" : true }, "storage" : 75600000 }, { "name" : "time_taken", "schema" : { "type" : "FLOAT64", "parameters" : { "logField" : "time-taken" }, "isOptional" : true }, "storage" : 1234.0 }, { "name" : "ip1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ip1" }, "isOptional" : true }, "storage" : "1.2.3.4" }, { "name" : "user", "schema" : { "type" : "STRING", "parameters" : { "logField" : "user" }, "isOptional" : true }, "storage" : "xyz" }, { "name" : "gp", "schema" : { "type" : "STRING", "parameters" : { "logField" : "gp" }, "isOptional" : true }, "storage" : "abc" }, { "name" : "id1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "id1" }, "isOptional" : true } }, { "name" : "res", "schema" : { "type" : "STRING", "parameters" : { "logField" : "res" }, "isOptional" : true }, "storage" : "OK" }, { "name" : "cat1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "cat1" }, "isOptional" : true }, "storage" : "cat" }, { "name" : "status", "schema" : { "type" : "STRING", "parameters" : { "logField" : "status" }, "isOptional" : true }, "storage" : "0" }, { "name" : "act", "schema" : { "type" : "STRING", "parameters" : { "logField" : "act" }, "isOptional" : true }, "storage" : "zyx" }, { "name" : "method", "schema" : { "type" : "STRING", "parameters" : { "logField" : "method" }, "isOptional" : true }, "storage" : "unk" }, { "name" : "ct", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ct" }, "isOptional" : true } }, { "name" : "scheme", "schema" : { "type" : "STRING", "parameters" : { "logField" : "scheme" }, "isOptional" : true }, "storage" : "pro" }, { "name" : "host", "schema" : { "type" : "STRING", "parameters" : { "logField" : "host" }, "isOptional" : true }, "storage" : "site" }, { "name" : "port", "schema" : { "type" : "STRING", "parameters" : { "logField" : "port" }, "isOptional" : true }, "storage" : "123" }, { "name" : "ext", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ext" }, "isOptional" : true } }, { "name" : "ua", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ua" }, "isOptional" : true } }, { "name" : "ip2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ip2" }, "isOptional" : true }, "storage" : "1.2.3.4" }, { "name" : "bytes1", "schema" : { "type" : "INT64", "parameters" : { "logField" : "bytes1" }, "isOptional" : true }, "storage" : 0 }, { "name" : "bytes2", "schema" : { "type" : "INT64", "parameters" : { "logField" : "bytes2" }, "isOptional" : true }, "storage" : 0 }, { "name" : "id2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "id2" }, "isOptional" : true } }, { "name" : "err1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "err1" }, "isOptional" : true }, "storage" : "none" }, { "name" : "err2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "err2" }, "isOptional" : true } }, { "name" : "err3", "schema" : { "type" : "STRING", "parameters" : { "logField" : "err3" }, "isOptional" : true } }, { "name" : "str", "schema" : { "type" : "STRING", "parameters" : { "logField" : "str" }, "isOptional" : true }, "storage" : "none" }, { "name" : "cr", "schema" : { "type" : "STRING", "parameters" : { "logField" : "cr" }, "isOptional" : true } }, { "name" : "cat2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "cat2" }, "isOptional" : true }, "storage" : "cat" }, { "name" : "datetime", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "logField" : "date,time" }, "isOptional" : true }, "storage" : 1571864400000 } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.elf" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1572979596824-0/input/FieldsMatch.elf" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 683 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 0 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1572979597000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.elf" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "valueSchema" : { "name" : "com.github.jcustenborder.kafka.connect.spooldir.LogEntry", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "date" : { "name" : "org.apache.kafka.connect.data.Date", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "date" }, "isOptional" : true }, "time" : { "name" : "org.apache.kafka.connect.data.Time", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "time" }, "isOptional" : true }, "time_taken" : { "type" : "FLOAT64", "parameters" : { "logField" : "time-taken" }, "isOptional" : true }, "ip1" : { "type" : "STRING", "parameters" : { "logField" : "ip1" }, "isOptional" : true }, "user" : { "type" : "STRING", "parameters" : { "logField" : "user" }, "isOptional" : true }, "gp" : { "type" : "STRING", "parameters" : { "logField" : "gp" }, "isOptional" : true }, "id1" : { "type" : "STRING", "parameters" : { "logField" : "id1" }, "isOptional" : true }, "res" : { "type" : "STRING", "parameters" : { "logField" : "res" }, "isOptional" : true }, "cat1" : { "type" : "STRING", "parameters" : { "logField" : "cat1" }, "isOptional" : true }, "status" : { "type" : "STRING", "parameters" : { "logField" : "status" }, "isOptional" : true }, "act" : { "type" : "STRING", "parameters" : { "logField" : "act" }, "isOptional" : true }, "method" : { "type" : "STRING", "parameters" : { "logField" : "method" }, "isOptional" : true }, "ct" : { "type" : "STRING", "parameters" : { "logField" : "ct" }, "isOptional" : true }, "scheme" : { "type" : "STRING", "parameters" : { "logField" : "scheme" }, "isOptional" : true }, "host" : { "type" : "STRING", "parameters" : { "logField" : "host" }, "isOptional" : true }, "port" : { "type" : "STRING", "parameters" : { "logField" : "port" }, "isOptional" : true }, "ext" : { "type" : "STRING", "parameters" : { "logField" : "ext" }, "isOptional" : true }, "ua" : { "type" : "STRING", "parameters" : { "logField" : "ua" }, "isOptional" : true }, "ip2" : { "type" : "STRING", "parameters" : { "logField" : "ip2" }, "isOptional" : true }, "bytes1" : { "type" : "INT64", "parameters" : { "logField" : "bytes1" }, "isOptional" : true }, "bytes2" : { "type" : "INT64", "parameters" : { "logField" : "bytes2" }, "isOptional" : true }, "id2" : { "type" : "STRING", "parameters" : { "logField" : "id2" }, "isOptional" : true }, "err1" : { "type" : "STRING", "parameters" : { "logField" : "err1" }, "isOptional" : true }, "err2" : { "type" : "STRING", "parameters" : { "logField" : "err2" }, "isOptional" : true }, "err3" : { "type" : "STRING", "parameters" : { "logField" : "err3" }, "isOptional" : true }, "str" : { "type" : "STRING", "parameters" : { "logField" : "str" }, "isOptional" : true }, "cr" : { "type" : "STRING", "parameters" : { "logField" : "cr" }, "isOptional" : true }, "cat2" : { "type" : "STRING", "parameters" : { "logField" : "cat2" }, "isOptional" : true }, "datetime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "logField" : "date,time" }, "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.github.jcustenborder.kafka.connect.spooldir.LogEntry", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "date" : { "name" : "org.apache.kafka.connect.data.Date", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "date" }, "isOptional" : true }, "time" : { "name" : "org.apache.kafka.connect.data.Time", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "time" }, "isOptional" : true }, "time_taken" : { "type" : "FLOAT64", "parameters" : { "logField" : "time-taken" }, "isOptional" : true }, "ip1" : { "type" : "STRING", "parameters" : { "logField" : "ip1" }, "isOptional" : true }, "user" : { "type" : "STRING", "parameters" : { "logField" : "user" }, "isOptional" : true }, "gp" : { "type" : "STRING", "parameters" : { "logField" : "gp" }, "isOptional" : true }, "id1" : { "type" : "STRING", "parameters" : { "logField" : "id1" }, "isOptional" : true }, "res" : { "type" : "STRING", "parameters" : { "logField" : "res" }, "isOptional" : true }, "cat1" : { "type" : "STRING", "parameters" : { "logField" : "cat1" }, "isOptional" : true }, "status" : { "type" : "STRING", "parameters" : { "logField" : "status" }, "isOptional" : true }, "act" : { "type" : "STRING", "parameters" : { "logField" : "act" }, "isOptional" : true }, "method" : { "type" : "STRING", "parameters" : { "logField" : "method" }, "isOptional" : true }, "ct" : { "type" : "STRING", "parameters" : { "logField" : "ct" }, "isOptional" : true }, "scheme" : { "type" : "STRING", "parameters" : { "logField" : "scheme" }, "isOptional" : true }, "host" : { "type" : "STRING", "parameters" : { "logField" : "host" }, "isOptional" : true }, "port" : { "type" : "STRING", "parameters" : { "logField" : "port" }, "isOptional" : true }, "ext" : { "type" : "STRING", "parameters" : { "logField" : "ext" }, "isOptional" : true }, "ua" : { "type" : "STRING", "parameters" : { "logField" : "ua" }, "isOptional" : true }, "ip2" : { "type" : "STRING", "parameters" : { "logField" : "ip2" }, "isOptional" : true }, "bytes1" : { "type" : "INT64", "parameters" : { "logField" : "bytes1" }, "isOptional" : true }, "bytes2" : { "type" : "INT64", "parameters" : { "logField" : "bytes2" }, "isOptional" : true }, "id2" : { "type" : "STRING", "parameters" : { "logField" : "id2" }, "isOptional" : true }, "err1" : { "type" : "STRING", "parameters" : { "logField" : "err1" }, "isOptional" : true }, "err2" : { "type" : "STRING", "parameters" : { "logField" : "err2" }, "isOptional" : true }, "err3" : { "type" : "STRING", "parameters" : { "logField" : "err3" }, "isOptional" : true }, "str" : { "type" : "STRING", "parameters" : { "logField" : "str" }, "isOptional" : true }, "cr" : { "type" : "STRING", "parameters" : { "logField" : "cr" }, "isOptional" : true }, "cat2" : { "type" : "STRING", "parameters" : { "logField" : "cat2" }, "isOptional" : true }, "datetime" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "logField" : "date,time" }, "isOptional" : true } } }, "fieldValues" : [ { "name" : "date", "schema" : { "name" : "org.apache.kafka.connect.data.Date", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "date" }, "isOptional" : true }, "storage" : 18192 }, { "name" : "time", "schema" : { "name" : "org.apache.kafka.connect.data.Time", "type" : "INT32", "version" : 1, "parameters" : { "logField" : "time" }, "isOptional" : true }, "storage" : 75600000 }, { "name" : "time_taken", "schema" : { "type" : "FLOAT64", "parameters" : { "logField" : "time-taken" }, "isOptional" : true }, "storage" : 123.0 }, { "name" : "ip1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ip1" }, "isOptional" : true }, "storage" : "1.2.3.4" }, { "name" : "user", "schema" : { "type" : "STRING", "parameters" : { "logField" : "user" }, "isOptional" : true }, "storage" : "xyz" }, { "name" : "gp", "schema" : { "type" : "STRING", "parameters" : { "logField" : "gp" }, "isOptional" : true }, "storage" : "abc" }, { "name" : "id1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "id1" }, "isOptional" : true } }, { "name" : "res", "schema" : { "type" : "STRING", "parameters" : { "logField" : "res" }, "isOptional" : true }, "storage" : "OK" }, { "name" : "cat1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "cat1" }, "isOptional" : true }, "storage" : "dog" }, { "name" : "status", "schema" : { "type" : "STRING", "parameters" : { "logField" : "status" }, "isOptional" : true }, "storage" : "0" }, { "name" : "act", "schema" : { "type" : "STRING", "parameters" : { "logField" : "act" }, "isOptional" : true }, "storage" : "zyx" }, { "name" : "method", "schema" : { "type" : "STRING", "parameters" : { "logField" : "method" }, "isOptional" : true }, "storage" : "unk" }, { "name" : "ct", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ct" }, "isOptional" : true } }, { "name" : "scheme", "schema" : { "type" : "STRING", "parameters" : { "logField" : "scheme" }, "isOptional" : true }, "storage" : "pro" }, { "name" : "host", "schema" : { "type" : "STRING", "parameters" : { "logField" : "host" }, "isOptional" : true }, "storage" : "site" }, { "name" : "port", "schema" : { "type" : "STRING", "parameters" : { "logField" : "port" }, "isOptional" : true }, "storage" : "123" }, { "name" : "ext", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ext" }, "isOptional" : true } }, { "name" : "ua", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ua" }, "isOptional" : true } }, { "name" : "ip2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "ip2" }, "isOptional" : true }, "storage" : "1.2.3.4" }, { "name" : "bytes1", "schema" : { "type" : "INT64", "parameters" : { "logField" : "bytes1" }, "isOptional" : true }, "storage" : 0 }, { "name" : "bytes2", "schema" : { "type" : "INT64", "parameters" : { "logField" : "bytes2" }, "isOptional" : true }, "storage" : 0 }, { "name" : "id2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "id2" }, "isOptional" : true } }, { "name" : "err1", "schema" : { "type" : "STRING", "parameters" : { "logField" : "err1" }, "isOptional" : true }, "storage" : "none" }, { "name" : "err2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "err2" }, "isOptional" : true } }, { "name" : "err3", "schema" : { "type" : "STRING", "parameters" : { "logField" : "err3" }, "isOptional" : true } }, { "name" : "str", "schema" : { "type" : "STRING", "parameters" : { "logField" : "str" }, "isOptional" : true }, "storage" : "none" }, { "name" : "cr", "schema" : { "type" : "STRING", "parameters" : { "logField" : "cr" }, "isOptional" : true } }, { "name" : "cat2", "schema" : { "type" : "STRING", "parameters" : { "logField" : "cat2" }, "isOptional" : true }, "storage" : "dog" }, { "name" : "datetime", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "parameters" : { "logField" : "date,time" }, "isOptional" : true }, "storage" : 1571864400000 } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.elf" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1572979596824-0/input/FieldsMatch.elf" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 683 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1572979597000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/DataHasMoreFields.data ================================================ { "id": 1, "first_name": "Jack", "last_name": "Garcia", "email": "jgarcia0@shop-pro.jp", "gender": "Male", "ip_address": "196.56.44.185", "last_login": "2015-09-30T15:29:03Z", "account_balance": 347.77, "country": "IT", "favorite_color": "#4a2313", "column11": "asdsa" } { "id": 2, "first_name": "John", "last_name": "Kim", "email": "jkim1@miibeian.gov.cn", "gender": "Male", "ip_address": "53.19.132.185", "last_login": "2015-11-14T10:34:09Z", "account_balance": 251.24, "country": "CZ", "favorite_color": "#3e56cf", "column11": "asd" } { "id": 3, "first_name": "Ashley", "last_name": "Austin", "email": "aaustin2@hatena.ne.jp", "gender": "Female", "ip_address": "21.164.37.9", "account_balance": 819.47, "country": "CN", "column11": "f" } { "id": 4, "first_name": "Jonathan", "last_name": "Mcdonald", "email": "jmcdonald3@amazon.co.uk", "gender": "Male", "ip_address": "188.172.42.140", "last_login": "2015-12-28T14:37:01Z", "account_balance": 868.38, "country": "ID", "favorite_color": "#1b1414", "column11": "as" } { "id": 5, "first_name": "Helen", "last_name": "Lane", "email": "hlane4@trellian.com", "gender": "Female", "ip_address": "159.171.138.190", "last_login": "2016-06-30T18:41:18Z", "account_balance": 398.97, "country": "TN", "column11": "g" } { "id": 6, "first_name": "Scott", "last_name": "Lopez", "email": "slopez5@google.co.jp", "gender": "Male", "ip_address": "86.194.226.35", "last_login": "2015-08-13T02:13:51Z", "account_balance": 322.99, "country": "BR", "column11": "g" } { "id": 7, "first_name": "Christine", "last_name": "Franklin", "email": "cfranklin6@reuters.com", "gender": "Female", "ip_address": "248.173.207.64", "last_login": "2015-12-22T11:29:57Z", "account_balance": 301.26, "country": "PH", "favorite_color": "#1d5e9d", "column11": "h" } { "id": 8, "first_name": "Helen", "last_name": "Andrews", "email": "handrews7@histats.com", "gender": "Female", "ip_address": "83.160.63.181", "last_login": "2016-03-06T11:41:10Z", "account_balance": 217.96, "country": "CU", "column11": "j" } { "id": 9, "first_name": "Stephanie", "last_name": "Gordon", "email": "sgordon8@goodreads.com", "gender": "Female", "ip_address": "193.143.42.212", "last_login": "2015-10-27T22:07:24Z", "account_balance": 495.8, "country": "CN", "column11": "f" } { "id": 10, "first_name": "Shirley", "last_name": "Andrews", "email": "sandrews9@flickr.com", "gender": "Female", "ip_address": "99.113.183.206", "last_login": "2015-11-07T11:12:52Z", "account_balance": 157.75, "country": "BR", "favorite_color": "#fc1da9", "column11": "s" } { "id": 11, "first_name": "Joshua", "last_name": "Reid", "email": "jreida@wikia.com", "gender": "Male", "ip_address": "197.96.118.164", "last_login": "2015-08-22T13:16:18Z", "account_balance": 431.8, "country": "CO", "favorite_color": "#6e3e36", "column11": "dfg" } { "id": 12, "first_name": "Frances", "last_name": "Parker", "email": "fparkerb@engadget.com", "gender": "Female", "ip_address": "226.237.57.25", "last_login": "2015-10-18T01:50:15Z", "account_balance": 188.21, "country": "BR", "favorite_color": "#73e909", "column11": "hg" } { "id": 13, "first_name": "Sharon", "last_name": "Lawson", "email": "slawsonc@bravesites.com", "gender": "Female", "ip_address": "198.189.134.106", "last_login": "2016-01-14T17:51:09Z", "account_balance": 206.73, "country": "VN", "column11": "s" } { "id": 14, "first_name": "Elizabeth", "last_name": "Wells", "email": "ewellsd@redcross.org", "gender": "Female", "ip_address": "120.108.59.206", "last_login": "2015-09-02T21:53:07Z", "account_balance": 499.48, "country": "CZ", "favorite_color": "#e9c943", "column11": "fgs" } { "id": 15, "first_name": "Norma", "last_name": "Wilson", "email": "nwilsone@google.com.br", "gender": "Female", "ip_address": "18.246.76.220", "last_login": "2015-09-27T02:10:48Z", "account_balance": -65.19, "country": "SE", "favorite_color": "#645119", "column11": "sdfgs" } { "id": 16, "first_name": "Joan", "last_name": "Watkins", "email": "jwatkinsf@yolasite.com", "gender": "Female", "ip_address": "240.27.33.114", "last_login": "2016-03-31T00:29:14Z", "account_balance": 264.23, "country": "PH", "column11": "sdfg" } { "id": 17, "first_name": "Gerald", "last_name": "Hamilton", "email": "ghamiltong@fc2.com", "gender": "Male", "ip_address": "182.75.62.95", "last_login": "2016-02-10T14:29:35Z", "account_balance": 309.26, "country": "ID", "column11": "sdfg" } { "id": 18, "first_name": "Paula", "last_name": "Taylor", "email": "ptaylorh@wikispaces.com", "gender": "Female", "ip_address": "245.74.203.0", "last_login": "2016-05-11T03:15:10Z", "account_balance": 927.45, "country": "CN", "column11": "sdfg" } { "id": 19, "first_name": "Carolyn", "last_name": "Burns", "email": "cburnsi@marketwatch.com", "gender": "Female", "ip_address": "180.243.11.10", "last_login": "2016-02-28T18:49:23Z", "account_balance": 752.76, "country": "NL", "column11": "dsf" } { "id": 20, "first_name": "Robin", "last_name": "Bennett", "email": "rbennettj@cdc.gov", "gender": "Female", "ip_address": "169.77.92.179", "last_login": "2016-02-15T01:06:44Z", "account_balance": 143.3, "country": "ID", "favorite_color": "#506128", "column11": "sdfgsdf" } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/DataHasMoreFields.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 0 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 0 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.json" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/DataHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FieldsMatch.data ================================================ { "id": 1, "first_name": "Jack", "last_name": "Garcia", "email": "jgarcia0@shop-pro.jp", "gender": "Male", "ip_address": "196.56.44.185", "last_login": "2015-09-30T15:29:03Z", "account_balance": 347.77, "country": "IT", "favorite_color": "#4a2313" } { "id": 2, "first_name": "John", "last_name": "Kim", "email": "jkim1@miibeian.gov.cn", "gender": "Male", "ip_address": "53.19.132.185", "last_login": "2015-11-14T10:34:09Z", "account_balance": 251.24, "country": "CZ", "favorite_color": "#3e56cf" } { "id": 3, "first_name": "Ashley", "last_name": "Austin", "email": "aaustin2@hatena.ne.jp", "gender": "Female", "ip_address": "21.164.37.9", "account_balance": 819.47, "country": "CN" } { "id": 4, "first_name": "Jonathan", "last_name": "Mcdonald", "email": "jmcdonald3@amazon.co.uk", "gender": "Male", "ip_address": "188.172.42.140", "last_login": "2015-12-28T14:37:01Z", "account_balance": 868.38, "country": "ID", "favorite_color": "#1b1414" } { "id": 5, "first_name": "Helen", "last_name": "Lane", "email": "hlane4@trellian.com", "gender": "Female", "ip_address": "159.171.138.190", "last_login": "2016-06-30T18:41:18Z", "account_balance": 398.97, "country": "TN" } { "id": 6, "first_name": "Scott", "last_name": "Lopez", "email": "slopez5@google.co.jp", "gender": "Male", "ip_address": "86.194.226.35", "last_login": "2015-08-13T02:13:51Z", "account_balance": 322.99, "country": "BR" } { "id": 7, "first_name": "Christine", "last_name": "Franklin", "email": "cfranklin6@reuters.com", "gender": "Female", "ip_address": "248.173.207.64", "last_login": "2015-12-22T11:29:57Z", "account_balance": 301.26, "country": "PH", "favorite_color": "#1d5e9d" } { "id": 8, "first_name": "Helen", "last_name": "Andrews", "email": "handrews7@histats.com", "gender": "Female", "ip_address": "83.160.63.181", "last_login": "2016-03-06T11:41:10Z", "account_balance": 217.96, "country": "CU" } { "id": 9, "first_name": "Stephanie", "last_name": "Gordon", "email": "sgordon8@goodreads.com", "gender": "Female", "ip_address": "193.143.42.212", "last_login": "2015-10-27T22:07:24Z", "account_balance": 495.8, "country": "CN" } { "id": 10, "first_name": "Shirley", "last_name": "Andrews", "email": "sandrews9@flickr.com", "gender": "Female", "ip_address": "99.113.183.206", "last_login": "2015-11-07T11:12:52Z", "account_balance": 157.75, "country": "BR", "favorite_color": "#fc1da9" } { "id": 11, "first_name": "Joshua", "last_name": "Reid", "email": "jreida@wikia.com", "gender": "Male", "ip_address": "197.96.118.164", "last_login": "2015-08-22T13:16:18Z", "account_balance": 431.8, "country": "CO", "favorite_color": "#6e3e36" } { "id": 12, "first_name": "Frances", "last_name": "Parker", "email": "fparkerb@engadget.com", "gender": "Female", "ip_address": "226.237.57.25", "last_login": "2015-10-18T01:50:15Z", "account_balance": 188.21, "country": "BR", "favorite_color": "#73e909" } { "id": 13, "first_name": "Sharon", "last_name": "Lawson", "email": "slawsonc@bravesites.com", "gender": "Female", "ip_address": "198.189.134.106", "last_login": "2016-01-14T17:51:09Z", "account_balance": 206.73, "country": "VN" } { "id": 14, "first_name": "Elizabeth", "last_name": "Wells", "email": "ewellsd@redcross.org", "gender": "Female", "ip_address": "120.108.59.206", "last_login": "2015-09-02T21:53:07Z", "account_balance": 499.48, "country": "CZ", "favorite_color": "#e9c943" } { "id": 15, "first_name": "Norma", "last_name": "Wilson", "email": "nwilsone@google.com.br", "gender": "Female", "ip_address": "18.246.76.220", "last_login": "2015-09-27T02:10:48Z", "account_balance": -65.19, "country": "SE", "favorite_color": "#645119" } { "id": 16, "first_name": "Joan", "last_name": "Watkins", "email": "jwatkinsf@yolasite.com", "gender": "Female", "ip_address": "240.27.33.114", "last_login": "2016-03-31T00:29:14Z", "account_balance": 264.23, "country": "PH" } { "id": 17, "first_name": "Gerald", "last_name": "Hamilton", "email": "ghamiltong@fc2.com", "gender": "Male", "ip_address": "182.75.62.95", "last_login": "2016-02-10T14:29:35Z", "account_balance": 309.26, "country": "ID" } { "id": 18, "first_name": "Paula", "last_name": "Taylor", "email": "ptaylorh@wikispaces.com", "gender": "Female", "ip_address": "245.74.203.0", "last_login": "2016-05-11T03:15:10Z", "account_balance": 927.45, "country": "CN" } { "id": 19, "first_name": "Carolyn", "last_name": "Burns", "email": "cburnsi@marketwatch.com", "gender": "Female", "ip_address": "180.243.11.10", "last_login": "2016-02-28T18:49:23Z", "account_balance": 752.76, "country": "NL" } { "id": 20, "first_name": "Robin", "last_name": "Bennett", "email": "rbennettj@cdc.gov", "gender": "Female", "ip_address": "169.77.92.179", "last_login": "2016-02-15T01:06:44Z", "account_balance": 143.3, "country": "ID", "favorite_color": "#506128" } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FieldsMatch.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 0 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 0 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FieldsMatch.json" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FileModeFieldFieldsMatch.data ================================================ { "id": 1, "first_name": "Jack", "last_name": "Garcia", "email": "jgarcia0@shop-pro.jp", "gender": "Male", "ip_address": "196.56.44.185", "last_login": "2015-09-30T15:29:03Z", "account_balance": 347.77, "country": "IT", "favorite_color": "#4a2313" } { "id": 2, "first_name": "John", "last_name": "Kim", "email": "jkim1@miibeian.gov.cn", "gender": "Male", "ip_address": "53.19.132.185", "last_login": "2015-11-14T10:34:09Z", "account_balance": 251.24, "country": "CZ", "favorite_color": "#3e56cf" } { "id": 3, "first_name": "Ashley", "last_name": "Austin", "email": "aaustin2@hatena.ne.jp", "gender": "Female", "ip_address": "21.164.37.9", "last_login": "2015-09-30T15:29:03Z", "account_balance": 819.47, "country": "CN" } { "id": 4, "first_name": "Jonathan", "last_name": "Mcdonald", "email": "jmcdonald3@amazon.co.uk", "gender": "Male", "ip_address": "188.172.42.140", "last_login": "2015-12-28T14:37:01Z", "account_balance": 868.38, "country": "ID", "favorite_color": "#1b1414" } { "id": 5, "first_name": "Helen", "last_name": "Lane", "email": "hlane4@trellian.com", "gender": "Female", "ip_address": "159.171.138.190", "last_login": "2016-06-30T18:41:18Z", "account_balance": 398.97, "country": "TN" } { "id": 6, "first_name": "Scott", "last_name": "Lopez", "email": "slopez5@google.co.jp", "gender": "Male", "ip_address": "86.194.226.35", "last_login": "2015-08-13T02:13:51Z", "account_balance": 322.99, "country": "BR" } { "id": 7, "first_name": "Christine", "last_name": "Franklin", "email": "cfranklin6@reuters.com", "gender": "Female", "ip_address": "248.173.207.64", "last_login": "2015-12-22T11:29:57Z", "account_balance": 301.26, "country": "PH", "favorite_color": "#1d5e9d" } { "id": 8, "first_name": "Helen", "last_name": "Andrews", "email": "handrews7@histats.com", "gender": "Female", "ip_address": "83.160.63.181", "last_login": "2016-03-06T11:41:10Z", "account_balance": 217.96, "country": "CU" } { "id": 9, "first_name": "Stephanie", "last_name": "Gordon", "email": "sgordon8@goodreads.com", "gender": "Female", "ip_address": "193.143.42.212", "last_login": "2015-10-27T22:07:24Z", "account_balance": 495.8, "country": "CN" } { "id": 10, "first_name": "Shirley", "last_name": "Andrews", "email": "sandrews9@flickr.com", "gender": "Female", "ip_address": "99.113.183.206", "last_login": "2015-11-07T11:12:52Z", "account_balance": 157.75, "country": "BR", "favorite_color": "#fc1da9" } { "id": 11, "first_name": "Joshua", "last_name": "Reid", "email": "jreida@wikia.com", "gender": "Male", "ip_address": "197.96.118.164", "last_login": "2015-08-22T13:16:18Z", "account_balance": 431.8, "country": "CO", "favorite_color": "#6e3e36" } { "id": 12, "first_name": "Frances", "last_name": "Parker", "email": "fparkerb@engadget.com", "gender": "Female", "ip_address": "226.237.57.25", "last_login": "2015-10-18T01:50:15Z", "account_balance": 188.21, "country": "BR", "favorite_color": "#73e909" } { "id": 13, "first_name": "Sharon", "last_name": "Lawson", "email": "slawsonc@bravesites.com", "gender": "Female", "ip_address": "198.189.134.106", "last_login": "2016-01-14T17:51:09Z", "account_balance": 206.73, "country": "VN" } { "id": 14, "first_name": "Elizabeth", "last_name": "Wells", "email": "ewellsd@redcross.org", "gender": "Female", "ip_address": "120.108.59.206", "last_login": "2015-09-02T21:53:07Z", "account_balance": 499.48, "country": "CZ", "favorite_color": "#e9c943" } { "id": 15, "first_name": "Norma", "last_name": "Wilson", "email": "nwilsone@google.com.br", "gender": "Female", "ip_address": "18.246.76.220", "last_login": "2015-09-27T02:10:48Z", "account_balance": -65.19, "country": "SE", "favorite_color": "#645119" } { "id": 16, "first_name": "Joan", "last_name": "Watkins", "email": "jwatkinsf@yolasite.com", "gender": "Female", "ip_address": "240.27.33.114", "last_login": "2016-03-31T00:29:14Z", "account_balance": 264.23, "country": "PH" } { "id": 17, "first_name": "Gerald", "last_name": "Hamilton", "email": "ghamiltong@fc2.com", "gender": "Male", "ip_address": "182.75.62.95", "last_login": "2016-02-10T14:29:35Z", "account_balance": 309.26, "country": "ID" } { "id": 18, "first_name": "Paula", "last_name": "Taylor", "email": "ptaylorh@wikispaces.com", "gender": "Female", "ip_address": "245.74.203.0", "last_login": "2016-05-11T03:15:10Z", "account_balance": 927.45, "country": "CN" } { "id": 19, "first_name": "Carolyn", "last_name": "Burns", "email": "cburnsi@marketwatch.com", "gender": "Female", "ip_address": "180.243.11.10", "last_login": "2016-02-28T18:49:23Z", "account_balance": 752.76, "country": "NL" } { "id": 20, "first_name": "Robin", "last_name": "Bennett", "email": "rbennettj@cdc.gov", "gender": "Female", "ip_address": "169.77.92.179", "last_login": "2016-02-15T01:06:44Z", "account_balance": 143.3, "country": "ID", "favorite_color": "#506128" } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/FileModeFieldFieldsMatch.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 0 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#4a2313" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 0 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#3e56cf" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1b1414" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#1d5e9d" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#fc1da9" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "FileModeFieldFieldsMatch.json" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "FileModeFieldFieldsMatch.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/FileModeFieldFieldsMatch.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4807 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/SchemaHasMoreFields.data ================================================ { "id": 1, "first_name": "Jack", "last_name": "Garcia", "email": "jgarcia0@shop-pro.jp", "gender": "Male", "ip_address": "196.56.44.185", "last_login": "2015-09-30T15:29:03Z", "account_balance": 347.77, "country": "IT" } { "id": 2, "first_name": "John", "last_name": "Kim", "email": "jkim1@miibeian.gov.cn", "gender": "Male", "ip_address": "53.19.132.185", "last_login": "2015-11-14T10:34:09Z", "account_balance": 251.24, "country": "CZ" } { "id": 3, "first_name": "Ashley", "last_name": "Austin", "email": "aaustin2@hatena.ne.jp", "gender": "Female", "ip_address": "21.164.37.9", "account_balance": 819.47, "country": "CN" } { "id": 4, "first_name": "Jonathan", "last_name": "Mcdonald", "email": "jmcdonald3@amazon.co.uk", "gender": "Male", "ip_address": "188.172.42.140", "last_login": "2015-12-28T14:37:01Z", "account_balance": 868.38, "country": "ID" } { "id": 5, "first_name": "Helen", "last_name": "Lane", "email": "hlane4@trellian.com", "gender": "Female", "ip_address": "159.171.138.190", "last_login": "2016-06-30T18:41:18Z", "account_balance": 398.97, "country": "TN" } { "id": 6, "first_name": "Scott", "last_name": "Lopez", "email": "slopez5@google.co.jp", "gender": "Male", "ip_address": "86.194.226.35", "last_login": "2015-08-13T02:13:51Z", "account_balance": 322.99, "country": "BR" } { "id": 7, "first_name": "Christine", "last_name": "Franklin", "email": "cfranklin6@reuters.com", "gender": "Female", "ip_address": "248.173.207.64", "last_login": "2015-12-22T11:29:57Z", "account_balance": 301.26, "country": "PH" } { "id": 8, "first_name": "Helen", "last_name": "Andrews", "email": "handrews7@histats.com", "gender": "Female", "ip_address": "83.160.63.181", "last_login": "2016-03-06T11:41:10Z", "account_balance": 217.96, "country": "CU" } { "id": 9, "first_name": "Stephanie", "last_name": "Gordon", "email": "sgordon8@goodreads.com", "gender": "Female", "ip_address": "193.143.42.212", "last_login": "2015-10-27T22:07:24Z", "account_balance": 495.8, "country": "CN" } { "id": 10, "first_name": "Shirley", "last_name": "Andrews", "email": "sandrews9@flickr.com", "gender": "Female", "ip_address": "99.113.183.206", "last_login": "2015-11-07T11:12:52Z", "account_balance": 157.75, "country": "BR" } { "id": 11, "first_name": "Joshua", "last_name": "Reid", "email": "jreida@wikia.com", "gender": "Male", "ip_address": "197.96.118.164", "last_login": "2015-08-22T13:16:18Z", "account_balance": 431.8, "country": "CO" } { "id": 12, "first_name": "Frances", "last_name": "Parker", "email": "fparkerb@engadget.com", "gender": "Female", "ip_address": "226.237.57.25", "last_login": "2015-10-18T01:50:15Z", "account_balance": 188.21, "country": "BR" } { "id": 13, "first_name": "Sharon", "last_name": "Lawson", "email": "slawsonc@bravesites.com", "gender": "Female", "ip_address": "198.189.134.106", "last_login": "2016-01-14T17:51:09Z", "account_balance": 206.73, "country": "VN" } { "id": 14, "first_name": "Elizabeth", "last_name": "Wells", "email": "ewellsd@redcross.org", "gender": "Female", "ip_address": "120.108.59.206", "last_login": "2015-09-02T21:53:07Z", "account_balance": 499.48, "country": "CZ" } { "id": 15, "first_name": "Norma", "last_name": "Wilson", "email": "nwilsone@google.com.br", "gender": "Female", "ip_address": "18.246.76.220", "last_login": "2015-09-27T02:10:48Z", "account_balance": -65.19, "country": "SE" } { "id": 16, "first_name": "Joan", "last_name": "Watkins", "email": "jwatkinsf@yolasite.com", "gender": "Female", "ip_address": "240.27.33.114", "last_login": "2016-03-31T00:29:14Z", "account_balance": 264.23, "country": "PH" } { "id": 17, "first_name": "Gerald", "last_name": "Hamilton", "email": "ghamiltong@fc2.com", "gender": "Male", "ip_address": "182.75.62.95", "last_login": "2016-02-10T14:29:35Z", "account_balance": 309.26, "country": "ID" } { "id": 18, "first_name": "Paula", "last_name": "Taylor", "email": "ptaylorh@wikispaces.com", "gender": "Female", "ip_address": "245.74.203.0", "last_login": "2016-05-11T03:15:10Z", "account_balance": 927.45, "country": "CN" } { "id": 19, "first_name": "Carolyn", "last_name": "Burns", "email": "cburnsi@marketwatch.com", "gender": "Female", "ip_address": "180.243.11.10", "last_login": "2016-02-28T18:49:23Z", "account_balance": 752.76, "country": "NL" } { "id": 20, "first_name": "Robin", "last_name": "Bennett", "email": "rbennettj@cdc.gov", "gender": "Female", "ip_address": "169.77.92.179", "last_login": "2016-02-15T01:06:44Z", "account_balance": 143.3, "country": "ID" } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/SchemaHasMoreFields.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 0 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jack" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Garcia" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jgarcia0@shop-pro.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "196.56.44.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443626943000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 347.77 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "IT" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 0 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "John" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Kim" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jkim1@miibeian.gov.cn" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "53.19.132.185" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1447497249000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 251.24 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Ashley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Austin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "aaustin2@hatena.ne.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "21.164.37.9" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true } }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 819.47 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Jonathan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Mcdonald" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jmcdonald3@amazon.co.uk" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "188.172.42.140" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1451313421000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 868.38 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lane" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "hlane4@trellian.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "159.171.138.190" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1467312078000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 398.97 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "TN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Scott" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lopez" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slopez5@google.co.jp" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "86.194.226.35" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1439432031000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 322.99 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Christine" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Franklin" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cfranklin6@reuters.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "248.173.207.64" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1450783797000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 301.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Helen" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "handrews7@histats.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "83.160.63.181" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1457264470000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 217.96 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CU" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Stephanie" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gordon" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sgordon8@goodreads.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "193.143.42.212" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445983644000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 495.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Shirley" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Andrews" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "sandrews9@flickr.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "99.113.183.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1446894772000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 157.75 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SchemaHasMoreFields.json" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SchemaHasMoreFields.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SchemaHasMoreFields.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4480 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/SourceOffset.data ================================================ { "id": 1, "first_name": "Jack", "last_name": "Garcia", "email": "jgarcia0@shop-pro.jp", "gender": "Male", "ip_address": "196.56.44.185", "last_login": "2015-09-30T15:29:03Z", "account_balance": 347.77, "country": "IT", "favorite_color": "#4a2313" } { "id": 2, "first_name": "John", "last_name": "Kim", "email": "jkim1@miibeian.gov.cn", "gender": "Male", "ip_address": "53.19.132.185", "last_login": "2015-11-14T10:34:09Z", "account_balance": 251.24, "country": "CZ", "favorite_color": "#3e56cf" } { "id": 3, "first_name": "Ashley", "last_name": "Austin", "email": "aaustin2@hatena.ne.jp", "gender": "Female", "ip_address": "21.164.37.9", "account_balance": 819.47, "country": "CN" } { "id": 4, "first_name": "Jonathan", "last_name": "Mcdonald", "email": "jmcdonald3@amazon.co.uk", "gender": "Male", "ip_address": "188.172.42.140", "last_login": "2015-12-28T14:37:01Z", "account_balance": 868.38, "country": "ID", "favorite_color": "#1b1414" } { "id": 5, "first_name": "Helen", "last_name": "Lane", "email": "hlane4@trellian.com", "gender": "Female", "ip_address": "159.171.138.190", "last_login": "2016-06-30T18:41:18Z", "account_balance": 398.97, "country": "TN" } { "id": 6, "first_name": "Scott", "last_name": "Lopez", "email": "slopez5@google.co.jp", "gender": "Male", "ip_address": "86.194.226.35", "last_login": "2015-08-13T02:13:51Z", "account_balance": 322.99, "country": "BR" } { "id": 7, "first_name": "Christine", "last_name": "Franklin", "email": "cfranklin6@reuters.com", "gender": "Female", "ip_address": "248.173.207.64", "last_login": "2015-12-22T11:29:57Z", "account_balance": 301.26, "country": "PH", "favorite_color": "#1d5e9d" } { "id": 8, "first_name": "Helen", "last_name": "Andrews", "email": "handrews7@histats.com", "gender": "Female", "ip_address": "83.160.63.181", "last_login": "2016-03-06T11:41:10Z", "account_balance": 217.96, "country": "CU" } { "id": 9, "first_name": "Stephanie", "last_name": "Gordon", "email": "sgordon8@goodreads.com", "gender": "Female", "ip_address": "193.143.42.212", "last_login": "2015-10-27T22:07:24Z", "account_balance": 495.8, "country": "CN" } { "id": 10, "first_name": "Shirley", "last_name": "Andrews", "email": "sandrews9@flickr.com", "gender": "Female", "ip_address": "99.113.183.206", "last_login": "2015-11-07T11:12:52Z", "account_balance": 157.75, "country": "BR", "favorite_color": "#fc1da9" } { "id": 11, "first_name": "Joshua", "last_name": "Reid", "email": "jreida@wikia.com", "gender": "Male", "ip_address": "197.96.118.164", "last_login": "2015-08-22T13:16:18Z", "account_balance": 431.8, "country": "CO", "favorite_color": "#6e3e36" } { "id": 12, "first_name": "Frances", "last_name": "Parker", "email": "fparkerb@engadget.com", "gender": "Female", "ip_address": "226.237.57.25", "last_login": "2015-10-18T01:50:15Z", "account_balance": 188.21, "country": "BR", "favorite_color": "#73e909" } { "id": 13, "first_name": "Sharon", "last_name": "Lawson", "email": "slawsonc@bravesites.com", "gender": "Female", "ip_address": "198.189.134.106", "last_login": "2016-01-14T17:51:09Z", "account_balance": 206.73, "country": "VN" } { "id": 14, "first_name": "Elizabeth", "last_name": "Wells", "email": "ewellsd@redcross.org", "gender": "Female", "ip_address": "120.108.59.206", "last_login": "2015-09-02T21:53:07Z", "account_balance": 499.48, "country": "CZ", "favorite_color": "#e9c943" } { "id": 15, "first_name": "Norma", "last_name": "Wilson", "email": "nwilsone@google.com.br", "gender": "Female", "ip_address": "18.246.76.220", "last_login": "2015-09-27T02:10:48Z", "account_balance": -65.19, "country": "SE", "favorite_color": "#645119" } { "id": 16, "first_name": "Joan", "last_name": "Watkins", "email": "jwatkinsf@yolasite.com", "gender": "Female", "ip_address": "240.27.33.114", "last_login": "2016-03-31T00:29:14Z", "account_balance": 264.23, "country": "PH" } { "id": 17, "first_name": "Gerald", "last_name": "Hamilton", "email": "ghamiltong@fc2.com", "gender": "Male", "ip_address": "182.75.62.95", "last_login": "2016-02-10T14:29:35Z", "account_balance": 309.26, "country": "ID" } { "id": 18, "first_name": "Paula", "last_name": "Taylor", "email": "ptaylorh@wikispaces.com", "gender": "Female", "ip_address": "245.74.203.0", "last_login": "2016-05-11T03:15:10Z", "account_balance": 927.45, "country": "CN" } { "id": 19, "first_name": "Carolyn", "last_name": "Burns", "email": "cburnsi@marketwatch.com", "gender": "Female", "ip_address": "180.243.11.10", "last_login": "2016-02-28T18:49:23Z", "account_balance": 752.76, "country": "NL" } { "id": 20, "first_name": "Robin", "last_name": "Bennett", "email": "rbennettj@cdc.gov", "gender": "Female", "ip_address": "169.77.92.179", "last_login": "2016-02-15T01:06:44Z", "account_balance": 143.3, "country": "ID", "favorite_color": "#506128" } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/json/SourceOffset.json ================================================ { "settings" : { "csv.first.row.as.header" : "true" }, "offset" : { "offset" : 10 }, "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "expected" : [ { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joshua" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Reid" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jreida@wikia.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "197.96.118.164" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1440249378000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 431.80 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CO" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#6e3e36" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Frances" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Parker" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "fparkerb@engadget.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "226.237.57.25" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1445133015000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 188.21 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "BR" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#73e909" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Sharon" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Lawson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "slawsonc@bravesites.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "198.189.134.106" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1452793869000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 206.73 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "VN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Elizabeth" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wells" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ewellsd@redcross.org" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "120.108.59.206" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1441230787000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 499.48 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CZ" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#e9c943" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Norma" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Wilson" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "nwilsone@google.com.br" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "18.246.76.220" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1443319848000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : -65.19 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "SE" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#645119" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Joan" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Watkins" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "jwatkinsf@yolasite.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "240.27.33.114" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1459384154000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 264.23 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "PH" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Gerald" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Hamilton" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ghamiltong@fc2.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Male" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "182.75.62.95" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455114575000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 309.26 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Paula" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Taylor" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ptaylorh@wikispaces.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "245.74.203.0" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1462936510000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 927.45 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "CN" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Carolyn" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Burns" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "cburnsi@marketwatch.com" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "180.243.11.10" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1456685363000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 752.76 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "NL" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true } } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] }, { "sourcePartition" : { "fileName" : "SourceOffset.json" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "keySchema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "key" : { "schema" : { "name" : "com.example.users.UserKey", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 } ] }, "valueSchema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "value" : { "schema" : { "name" : "com.example.users.User", "type" : "STRUCT", "isOptional" : false, "fieldSchemas" : { "id" : { "type" : "INT64", "isOptional" : false }, "first_name" : { "type" : "STRING", "isOptional" : true }, "last_name" : { "type" : "STRING", "isOptional" : true }, "email" : { "type" : "STRING", "isOptional" : true }, "gender" : { "type" : "STRING", "isOptional" : true }, "ip_address" : { "type" : "STRING", "isOptional" : true }, "last_login" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "account_balance" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "country" : { "type" : "STRING", "isOptional" : true }, "favorite_color" : { "type" : "STRING", "isOptional" : true } } }, "fieldValues" : [ { "name" : "id", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 20 }, { "name" : "first_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Robin" }, { "name" : "last_name", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Bennett" }, { "name" : "email", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "rbennettj@cdc.gov" }, { "name" : "gender", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "Female" }, { "name" : "ip_address", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "169.77.92.179" }, { "name" : "last_login", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : true }, "storage" : 1455498404000 }, { "name" : "account_balance", "schema" : { "name" : "org.apache.kafka.connect.data.Decimal", "type" : "BYTES", "version" : 1, "parameters" : { "scale" : "2" }, "isOptional" : true }, "storage" : 143.30 }, { "name" : "country", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "ID" }, { "name" : "favorite_color", "schema" : { "type" : "STRING", "isOptional" : true }, "storage" : "#506128" } ] }, "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "SourceOffset.json" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925242635-0/input/SourceOffset.json" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4770 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925242000 } ] } ] } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/schemalessjson/DataHasMoreFields.data ================================================ { "id": 1, "first_name": "Jack", "last_name": "Garcia", "email": "jgarcia0@shop-pro.jp", "gender": "Male", "ip_address": "196.56.44.185", "last_login": "2015-09-30T15:29:03Z", "account_balance": 347.77, "country": "IT", "favorite_color": "#4a2313", "column11": "asdsa" } { "id": 2, "first_name": "John", "last_name": "Kim", "email": "jkim1@miibeian.gov.cn", "gender": "Male", "ip_address": "53.19.132.185", "last_login": "2015-11-14T10:34:09Z", "account_balance": 251.24, "country": "CZ", "favorite_color": "#3e56cf", "column11": "asd" } { "id": 3, "first_name": "Ashley", "last_name": "Austin", "email": "aaustin2@hatena.ne.jp", "gender": "Female", "ip_address": "21.164.37.9", "account_balance": 819.47, "country": "CN", "column11": "f" } { "id": 4, "first_name": "Jonathan", "last_name": "Mcdonald", "email": "jmcdonald3@amazon.co.uk", "gender": "Male", "ip_address": "188.172.42.140", "last_login": "2015-12-28T14:37:01Z", "account_balance": 868.38, "country": "ID", "favorite_color": "#1b1414", "column11": "as" } { "id": 5, "first_name": "Helen", "last_name": "Lane", "email": "hlane4@trellian.com", "gender": "Female", "ip_address": "159.171.138.190", "last_login": "2016-06-30T18:41:18Z", "account_balance": 398.97, "country": "TN", "column11": "g" } { "id": 6, "first_name": "Scott", "last_name": "Lopez", "email": "slopez5@google.co.jp", "gender": "Male", "ip_address": "86.194.226.35", "last_login": "2015-08-13T02:13:51Z", "account_balance": 322.99, "country": "BR", "column11": "g" } { "id": 7, "first_name": "Christine", "last_name": "Franklin", "email": "cfranklin6@reuters.com", "gender": "Female", "ip_address": "248.173.207.64", "last_login": "2015-12-22T11:29:57Z", "account_balance": 301.26, "country": "PH", "favorite_color": "#1d5e9d", "column11": "h" } { "id": 8, "first_name": "Helen", "last_name": "Andrews", "email": "handrews7@histats.com", "gender": "Female", "ip_address": "83.160.63.181", "last_login": "2016-03-06T11:41:10Z", "account_balance": 217.96, "country": "CU", "column11": "j" } { "id": 9, "first_name": "Stephanie", "last_name": "Gordon", "email": "sgordon8@goodreads.com", "gender": "Female", "ip_address": "193.143.42.212", "last_login": "2015-10-27T22:07:24Z", "account_balance": 495.8, "country": "CN", "column11": "f" } { "id": 10, "first_name": "Shirley", "last_name": "Andrews", "email": "sandrews9@flickr.com", "gender": "Female", "ip_address": "99.113.183.206", "last_login": "2015-11-07T11:12:52Z", "account_balance": 157.75, "country": "BR", "favorite_color": "#fc1da9", "column11": "s" } { "id": 11, "first_name": "Joshua", "last_name": "Reid", "email": "jreida@wikia.com", "gender": "Male", "ip_address": "197.96.118.164", "last_login": "2015-08-22T13:16:18Z", "account_balance": 431.8, "country": "CO", "favorite_color": "#6e3e36", "column11": "dfg" } { "id": 12, "first_name": "Frances", "last_name": "Parker", "email": "fparkerb@engadget.com", "gender": "Female", "ip_address": "226.237.57.25", "last_login": "2015-10-18T01:50:15Z", "account_balance": 188.21, "country": "BR", "favorite_color": "#73e909", "column11": "hg" } { "id": 13, "first_name": "Sharon", "last_name": "Lawson", "email": "slawsonc@bravesites.com", "gender": "Female", "ip_address": "198.189.134.106", "last_login": "2016-01-14T17:51:09Z", "account_balance": 206.73, "country": "VN", "column11": "s" } { "id": 14, "first_name": "Elizabeth", "last_name": "Wells", "email": "ewellsd@redcross.org", "gender": "Female", "ip_address": "120.108.59.206", "last_login": "2015-09-02T21:53:07Z", "account_balance": 499.48, "country": "CZ", "favorite_color": "#e9c943", "column11": "fgs" } { "id": 15, "first_name": "Norma", "last_name": "Wilson", "email": "nwilsone@google.com.br", "gender": "Female", "ip_address": "18.246.76.220", "last_login": "2015-09-27T02:10:48Z", "account_balance": -65.19, "country": "SE", "favorite_color": "#645119", "column11": "sdfgs" } { "id": 16, "first_name": "Joan", "last_name": "Watkins", "email": "jwatkinsf@yolasite.com", "gender": "Female", "ip_address": "240.27.33.114", "last_login": "2016-03-31T00:29:14Z", "account_balance": 264.23, "country": "PH", "column11": "sdfg" } { "id": 17, "first_name": "Gerald", "last_name": "Hamilton", "email": "ghamiltong@fc2.com", "gender": "Male", "ip_address": "182.75.62.95", "last_login": "2016-02-10T14:29:35Z", "account_balance": 309.26, "country": "ID", "column11": "sdfg" } { "id": 18, "first_name": "Paula", "last_name": "Taylor", "email": "ptaylorh@wikispaces.com", "gender": "Female", "ip_address": "245.74.203.0", "last_login": "2016-05-11T03:15:10Z", "account_balance": 927.45, "country": "CN", "column11": "sdfg" } { "id": 19, "first_name": "Carolyn", "last_name": "Burns", "email": "cburnsi@marketwatch.com", "gender": "Female", "ip_address": "180.243.11.10", "last_login": "2016-02-28T18:49:23Z", "account_balance": 752.76, "country": "NL", "column11": "dsf" } { "id": 20, "first_name": "Robin", "last_name": "Bennett", "email": "rbennettj@cdc.gov", "gender": "Female", "ip_address": "169.77.92.179", "last_login": "2016-02-15T01:06:44Z", "account_balance": 143.3, "country": "ID", "favorite_color": "#506128", "column11": "sdfgsdf" } ================================================ FILE: src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/schemalessjson/DataHasMoreFields.json ================================================ { "settings" : { }, "offset" : { }, "expected" : [ { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 0 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 1,\n \"first_name\" : \"Jack\",\n \"last_name\" : \"Garcia\",\n \"email\" : \"jgarcia0@shop-pro.jp\",\n \"gender\" : \"Male\",\n \"ip_address\" : \"196.56.44.185\",\n \"last_login\" : \"2015-09-30T15:29:03Z\",\n \"account_balance\" : 347.77,\n \"country\" : \"IT\",\n \"favorite_color\" : \"#4a2313\",\n \"column11\" : \"asdsa\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 0 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 1 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 2,\n \"first_name\" : \"John\",\n \"last_name\" : \"Kim\",\n \"email\" : \"jkim1@miibeian.gov.cn\",\n \"gender\" : \"Male\",\n \"ip_address\" : \"53.19.132.185\",\n \"last_login\" : \"2015-11-14T10:34:09Z\",\n \"account_balance\" : 251.24,\n \"country\" : \"CZ\",\n \"favorite_color\" : \"#3e56cf\",\n \"column11\" : \"asd\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 1 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 2 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 3,\n \"first_name\" : \"Ashley\",\n \"last_name\" : \"Austin\",\n \"email\" : \"aaustin2@hatena.ne.jp\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"21.164.37.9\",\n \"account_balance\" : 819.47,\n \"country\" : \"CN\",\n \"column11\" : \"f\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 2 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 3 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 4,\n \"first_name\" : \"Jonathan\",\n \"last_name\" : \"Mcdonald\",\n \"email\" : \"jmcdonald3@amazon.co.uk\",\n \"gender\" : \"Male\",\n \"ip_address\" : \"188.172.42.140\",\n \"last_login\" : \"2015-12-28T14:37:01Z\",\n \"account_balance\" : 868.38,\n \"country\" : \"ID\",\n \"favorite_color\" : \"#1b1414\",\n \"column11\" : \"as\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 3 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 4 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 5,\n \"first_name\" : \"Helen\",\n \"last_name\" : \"Lane\",\n \"email\" : \"hlane4@trellian.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"159.171.138.190\",\n \"last_login\" : \"2016-06-30T18:41:18Z\",\n \"account_balance\" : 398.97,\n \"country\" : \"TN\",\n \"column11\" : \"g\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 4 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 5 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 6,\n \"first_name\" : \"Scott\",\n \"last_name\" : \"Lopez\",\n \"email\" : \"slopez5@google.co.jp\",\n \"gender\" : \"Male\",\n \"ip_address\" : \"86.194.226.35\",\n \"last_login\" : \"2015-08-13T02:13:51Z\",\n \"account_balance\" : 322.99,\n \"country\" : \"BR\",\n \"column11\" : \"g\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 6 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 7,\n \"first_name\" : \"Christine\",\n \"last_name\" : \"Franklin\",\n \"email\" : \"cfranklin6@reuters.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"248.173.207.64\",\n \"last_login\" : \"2015-12-22T11:29:57Z\",\n \"account_balance\" : 301.26,\n \"country\" : \"PH\",\n \"favorite_color\" : \"#1d5e9d\",\n \"column11\" : \"h\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 6 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 7 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 8,\n \"first_name\" : \"Helen\",\n \"last_name\" : \"Andrews\",\n \"email\" : \"handrews7@histats.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"83.160.63.181\",\n \"last_login\" : \"2016-03-06T11:41:10Z\",\n \"account_balance\" : 217.96,\n \"country\" : \"CU\",\n \"column11\" : \"j\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 7 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 8 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 9,\n \"first_name\" : \"Stephanie\",\n \"last_name\" : \"Gordon\",\n \"email\" : \"sgordon8@goodreads.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"193.143.42.212\",\n \"last_login\" : \"2015-10-27T22:07:24Z\",\n \"account_balance\" : 495.8,\n \"country\" : \"CN\",\n \"column11\" : \"f\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 8 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 9 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 10,\n \"first_name\" : \"Shirley\",\n \"last_name\" : \"Andrews\",\n \"email\" : \"sandrews9@flickr.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"99.113.183.206\",\n \"last_login\" : \"2015-11-07T11:12:52Z\",\n \"account_balance\" : 157.75,\n \"country\" : \"BR\",\n \"favorite_color\" : \"#fc1da9\",\n \"column11\" : \"s\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 9 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 10 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 11,\n \"first_name\" : \"Joshua\",\n \"last_name\" : \"Reid\",\n \"email\" : \"jreida@wikia.com\",\n \"gender\" : \"Male\",\n \"ip_address\" : \"197.96.118.164\",\n \"last_login\" : \"2015-08-22T13:16:18Z\",\n \"account_balance\" : 431.8,\n \"country\" : \"CO\",\n \"favorite_color\" : \"#6e3e36\",\n \"column11\" : \"dfg\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 10 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 11 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 12,\n \"first_name\" : \"Frances\",\n \"last_name\" : \"Parker\",\n \"email\" : \"fparkerb@engadget.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"226.237.57.25\",\n \"last_login\" : \"2015-10-18T01:50:15Z\",\n \"account_balance\" : 188.21,\n \"country\" : \"BR\",\n \"favorite_color\" : \"#73e909\",\n \"column11\" : \"hg\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 11 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 12 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 13,\n \"first_name\" : \"Sharon\",\n \"last_name\" : \"Lawson\",\n \"email\" : \"slawsonc@bravesites.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"198.189.134.106\",\n \"last_login\" : \"2016-01-14T17:51:09Z\",\n \"account_balance\" : 206.73,\n \"country\" : \"VN\",\n \"column11\" : \"s\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 12 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 13 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 14,\n \"first_name\" : \"Elizabeth\",\n \"last_name\" : \"Wells\",\n \"email\" : \"ewellsd@redcross.org\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"120.108.59.206\",\n \"last_login\" : \"2015-09-02T21:53:07Z\",\n \"account_balance\" : 499.48,\n \"country\" : \"CZ\",\n \"favorite_color\" : \"#e9c943\",\n \"column11\" : \"fgs\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 13 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 14 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 15,\n \"first_name\" : \"Norma\",\n \"last_name\" : \"Wilson\",\n \"email\" : \"nwilsone@google.com.br\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"18.246.76.220\",\n \"last_login\" : \"2015-09-27T02:10:48Z\",\n \"account_balance\" : -65.19,\n \"country\" : \"SE\",\n \"favorite_color\" : \"#645119\",\n \"column11\" : \"sdfgs\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 14 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 15 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 16,\n \"first_name\" : \"Joan\",\n \"last_name\" : \"Watkins\",\n \"email\" : \"jwatkinsf@yolasite.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"240.27.33.114\",\n \"last_login\" : \"2016-03-31T00:29:14Z\",\n \"account_balance\" : 264.23,\n \"country\" : \"PH\",\n \"column11\" : \"sdfg\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 15 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 16 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 17,\n \"first_name\" : \"Gerald\",\n \"last_name\" : \"Hamilton\",\n \"email\" : \"ghamiltong@fc2.com\",\n \"gender\" : \"Male\",\n \"ip_address\" : \"182.75.62.95\",\n \"last_login\" : \"2016-02-10T14:29:35Z\",\n \"account_balance\" : 309.26,\n \"country\" : \"ID\",\n \"column11\" : \"sdfg\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 16 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 17 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 18,\n \"first_name\" : \"Paula\",\n \"last_name\" : \"Taylor\",\n \"email\" : \"ptaylorh@wikispaces.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"245.74.203.0\",\n \"last_login\" : \"2016-05-11T03:15:10Z\",\n \"account_balance\" : 927.45,\n \"country\" : \"CN\",\n \"column11\" : \"sdfg\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 17 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 18 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 19,\n \"first_name\" : \"Carolyn\",\n \"last_name\" : \"Burns\",\n \"email\" : \"cburnsi@marketwatch.com\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"180.243.11.10\",\n \"last_login\" : \"2016-02-28T18:49:23Z\",\n \"account_balance\" : 752.76,\n \"country\" : \"NL\",\n \"column11\" : \"dsf\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 18 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] }, { "sourcePartition" : { "fileName" : "DataHasMoreFields.schemalessjson" }, "sourceOffset" : { "offset" : 19 }, "topic" : "testing", "valueSchema" : { "type" : "STRING", "isOptional" : false }, "value" : "{\n \"id\" : 20,\n \"first_name\" : \"Robin\",\n \"last_name\" : \"Bennett\",\n \"email\" : \"rbennettj@cdc.gov\",\n \"gender\" : \"Female\",\n \"ip_address\" : \"169.77.92.179\",\n \"last_login\" : \"2016-02-15T01:06:44Z\",\n \"account_balance\" : 143.3,\n \"country\" : \"ID\",\n \"favorite_color\" : \"#506128\",\n \"column11\" : \"sdfgsdf\"\n}", "headers" : [ { "name" : "file.name", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "DataHasMoreFields.schemalessjson" }, { "name" : "file.path", "schema" : { "type" : "STRING", "isOptional" : false }, "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925238742-0/input/DataHasMoreFields.schemalessjson" }, { "name" : "file.length", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 5153 }, { "name" : "file.offset", "schema" : { "type" : "INT64", "isOptional" : false }, "storage" : 19 }, { "name" : "file.last.modified", "schema" : { "name" : "org.apache.kafka.connect.data.Timestamp", "type" : "INT64", "version" : 1, "isOptional" : false }, "storage" : 1559925239000 } ] } ] } ================================================ FILE: src/test/resources/logback.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n