Showing preview only (5,579K chars total). Download the full file or copy to clipboard to get everything.
Repository: alibaba/DataX
Branch: master
Commit: 60ea07b73086
Files: 1445
Total size: 5.0 MB
Directory structure:
gitextract_n5eqdejp/
├── .gitignore
├── NOTICE
├── README.md
├── adbmysqlwriter/
│ ├── doc/
│ │ └── adbmysqlwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── adbmysqlwriter/
│ │ └── AdbMysqlWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── adbpgwriter/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── doc/
│ │ └── adbpgwriter.md
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── adbpgwriter/
│ │ ├── AdbpgWriter.java
│ │ ├── copy/
│ │ │ ├── Adb4pgClientProxy.java
│ │ │ └── AdbProxy.java
│ │ ├── package-info.java
│ │ └── util/
│ │ ├── Adb4pgUtil.java
│ │ ├── Constant.java
│ │ └── Key.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── adswriter/
│ ├── doc/
│ │ └── adswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── adswriter/
│ │ ├── AdsException.java
│ │ ├── AdsWriter.java
│ │ ├── AdsWriterErrorCode.java
│ │ ├── ads/
│ │ │ ├── ColumnDataType.java
│ │ │ ├── ColumnInfo.java
│ │ │ ├── TableInfo.java
│ │ │ └── package-info.java
│ │ ├── insert/
│ │ │ ├── AdsClientProxy.java
│ │ │ ├── AdsInsertProxy.java
│ │ │ ├── AdsInsertUtil.java
│ │ │ ├── AdsProxy.java
│ │ │ └── OperationType.java
│ │ ├── load/
│ │ │ ├── AdsHelper.java
│ │ │ ├── TableMetaHelper.java
│ │ │ └── TransferProjectConf.java
│ │ ├── odps/
│ │ │ ├── DataType.java
│ │ │ ├── FieldSchema.java
│ │ │ ├── TableMeta.java
│ │ │ └── package-info.java
│ │ ├── package-info.java
│ │ └── util/
│ │ ├── AdsUtil.java
│ │ ├── Constant.java
│ │ └── Key.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── cassandrareader/
│ ├── doc/
│ │ └── cassandrareader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── cassandrareader/
│ │ ├── CassandraReader.java
│ │ ├── CassandraReaderErrorCode.java
│ │ ├── CassandraReaderHelper.java
│ │ ├── Key.java
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ └── LocalStrings_zh_TW.properties
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── cassandrawriter/
│ ├── doc/
│ │ └── cassandrawriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── cassandrawriter/
│ │ ├── CassandraWriter.java
│ │ ├── CassandraWriterErrorCode.java
│ │ ├── CassandraWriterHelper.java
│ │ ├── Key.java
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ └── LocalStrings_zh_TW.properties
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── clickhousereader/
│ ├── doc/
│ │ └── clickhousereader.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ └── clickhousereader/
│ │ │ └── ClickhouseReader.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ └── resources/
│ ├── basic1.json
│ └── basic1.sql
├── clickhousewriter/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── clickhousewriter/
│ │ ├── ClickhouseWriter.java
│ │ └── ClickhouseWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── common/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── common/
│ ├── base/
│ │ └── BaseObject.java
│ ├── constant/
│ │ ├── CommonConstant.java
│ │ └── PluginType.java
│ ├── element/
│ │ ├── BoolColumn.java
│ │ ├── BytesColumn.java
│ │ ├── Column.java
│ │ ├── ColumnCast.java
│ │ ├── DateColumn.java
│ │ ├── DoubleColumn.java
│ │ ├── LongColumn.java
│ │ ├── OverFlowUtil.java
│ │ ├── Record.java
│ │ └── StringColumn.java
│ ├── exception/
│ │ ├── CommonErrorCode.java
│ │ ├── DataXException.java
│ │ └── ExceptionTracker.java
│ ├── plugin/
│ │ ├── AbstractJobPlugin.java
│ │ ├── AbstractPlugin.java
│ │ ├── AbstractTaskPlugin.java
│ │ ├── JobPluginCollector.java
│ │ ├── PluginCollector.java
│ │ ├── Pluginable.java
│ │ ├── RecordReceiver.java
│ │ ├── RecordSender.java
│ │ └── TaskPluginCollector.java
│ ├── spi/
│ │ ├── ErrorCode.java
│ │ ├── Hook.java
│ │ ├── Reader.java
│ │ └── Writer.java
│ ├── statistics/
│ │ ├── PerfRecord.java
│ │ ├── PerfTrace.java
│ │ └── VMInfo.java
│ └── util/
│ ├── Configuration.java
│ ├── ConfigurationUtil.java
│ ├── DESCipher.java
│ ├── DataXCaseEnvUtil.java
│ ├── FilterUtil.java
│ ├── HostUtils.java
│ ├── LimitLogger.java
│ ├── ListUtil.java
│ ├── LocalStrings.properties
│ ├── LocalStrings_en_US.properties
│ ├── LocalStrings_ja_JP.properties
│ ├── LocalStrings_zh_CN.properties
│ ├── LocalStrings_zh_HK.properties
│ ├── LocalStrings_zh_TW.properties
│ ├── LoggerFunction.java
│ ├── MessageSource.java
│ ├── RangeSplitUtil.java
│ ├── RetryUtil.java
│ └── StrUtil.java
├── core/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── conf/
│ │ ├── .secret.properties
│ │ ├── core.json
│ │ └── logback.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ ├── core/
│ │ │ ├── AbstractContainer.java
│ │ │ ├── Engine.java
│ │ │ ├── LocalStrings.properties
│ │ │ ├── LocalStrings_en_US.properties
│ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ ├── container/
│ │ │ │ └── util/
│ │ │ │ ├── HookInvoker.java
│ │ │ │ └── JobAssignUtil.java
│ │ │ ├── job/
│ │ │ │ ├── JobContainer.java
│ │ │ │ ├── meta/
│ │ │ │ │ ├── ExecuteMode.java
│ │ │ │ │ └── State.java
│ │ │ │ └── scheduler/
│ │ │ │ ├── AbstractScheduler.java
│ │ │ │ └── processinner/
│ │ │ │ ├── ProcessInnerScheduler.java
│ │ │ │ └── StandAloneScheduler.java
│ │ │ ├── statistics/
│ │ │ │ ├── communication/
│ │ │ │ │ ├── Communication.java
│ │ │ │ │ ├── CommunicationTool.java
│ │ │ │ │ └── LocalTGCommunicationManager.java
│ │ │ │ ├── container/
│ │ │ │ │ ├── collector/
│ │ │ │ │ │ ├── AbstractCollector.java
│ │ │ │ │ │ └── ProcessInnerCollector.java
│ │ │ │ │ ├── communicator/
│ │ │ │ │ │ ├── AbstractContainerCommunicator.java
│ │ │ │ │ │ ├── job/
│ │ │ │ │ │ │ └── StandAloneJobContainerCommunicator.java
│ │ │ │ │ │ └── taskgroup/
│ │ │ │ │ │ ├── AbstractTGContainerCommunicator.java
│ │ │ │ │ │ └── StandaloneTGContainerCommunicator.java
│ │ │ │ │ └── report/
│ │ │ │ │ ├── AbstractReporter.java
│ │ │ │ │ └── ProcessInnerReporter.java
│ │ │ │ └── plugin/
│ │ │ │ ├── DefaultJobPluginCollector.java
│ │ │ │ └── task/
│ │ │ │ ├── AbstractTaskPluginCollector.java
│ │ │ │ ├── HttpPluginCollector.java
│ │ │ │ ├── StdoutPluginCollector.java
│ │ │ │ └── util/
│ │ │ │ └── DirtyRecord.java
│ │ │ ├── taskgroup/
│ │ │ │ ├── TaskGroupContainer.java
│ │ │ │ ├── TaskMonitor.java
│ │ │ │ └── runner/
│ │ │ │ ├── AbstractRunner.java
│ │ │ │ ├── ReaderRunner.java
│ │ │ │ ├── TaskGroupContainerRunner.java
│ │ │ │ └── WriterRunner.java
│ │ │ ├── transport/
│ │ │ │ ├── channel/
│ │ │ │ │ ├── Channel.java
│ │ │ │ │ └── memory/
│ │ │ │ │ └── MemoryChannel.java
│ │ │ │ ├── exchanger/
│ │ │ │ │ ├── BufferedRecordExchanger.java
│ │ │ │ │ ├── BufferedRecordTransformerExchanger.java
│ │ │ │ │ ├── RecordExchanger.java
│ │ │ │ │ └── TransformerExchanger.java
│ │ │ │ ├── record/
│ │ │ │ │ ├── DefaultRecord.java
│ │ │ │ │ └── TerminateRecord.java
│ │ │ │ └── transformer/
│ │ │ │ ├── ComplexTransformerProxy.java
│ │ │ │ ├── DigestTransformer.java
│ │ │ │ ├── FilterTransformer.java
│ │ │ │ ├── GroovyTransformer.java
│ │ │ │ ├── GroovyTransformerStaticUtil.java
│ │ │ │ ├── PadTransformer.java
│ │ │ │ ├── ReplaceTransformer.java
│ │ │ │ ├── SubstrTransformer.java
│ │ │ │ ├── TransformerErrorCode.java
│ │ │ │ ├── TransformerExecution.java
│ │ │ │ ├── TransformerExecutionParas.java
│ │ │ │ ├── TransformerInfo.java
│ │ │ │ └── TransformerRegistry.java
│ │ │ └── util/
│ │ │ ├── ClassSize.java
│ │ │ ├── ClassUtil.java
│ │ │ ├── ConfigParser.java
│ │ │ ├── ConfigurationValidate.java
│ │ │ ├── ErrorRecordChecker.java
│ │ │ ├── ExceptionTracker.java
│ │ │ ├── FrameworkErrorCode.java
│ │ │ ├── HttpClientUtil.java
│ │ │ ├── LocalStrings.properties
│ │ │ ├── LocalStrings_en_US.properties
│ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ ├── SecretUtil.java
│ │ │ ├── TransformerUtil.java
│ │ │ └── container/
│ │ │ ├── ClassLoaderSwapper.java
│ │ │ ├── CoreConstant.java
│ │ │ ├── JarLoader.java
│ │ │ └── LoadUtil.java
│ │ └── dataxservice/
│ │ └── face/
│ │ └── domain/
│ │ └── enums/
│ │ ├── EnumStrVal.java
│ │ ├── EnumVal.java
│ │ ├── ExecuteMode.java
│ │ └── State.java
│ ├── job/
│ │ └── job.json
│ └── script/
│ └── Readme.md
├── databendwriter/
│ ├── doc/
│ │ ├── databendwriter-CN.md
│ │ └── databendwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── databendwriter/
│ │ ├── DatabendWriter.java
│ │ ├── DatabendWriterErrorCode.java
│ │ └── util/
│ │ └── DatabendWriterUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── datahubreader/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── datahubreader/
│ │ ├── Constant.java
│ │ ├── DatahubClientHelper.java
│ │ ├── DatahubReader.java
│ │ ├── DatahubReaderErrorCode.java
│ │ ├── DatahubReaderUtils.java
│ │ ├── DatahubWriterErrorCode.java
│ │ ├── Key.java
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ └── LocalStrings_zh_TW.properties
│ └── resources/
│ ├── job_config_template.json
│ └── plugin.json
├── datahubwriter/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── datahubwriter/
│ │ ├── DatahubClientHelper.java
│ │ ├── DatahubWriter.java
│ │ ├── DatahubWriterErrorCode.java
│ │ ├── Key.java
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ └── LocalStrings_zh_TW.properties
│ └── resources/
│ ├── job_config_template.json
│ └── plugin.json
├── datax-example/
│ ├── datax-example-core/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── java/
│ │ │ │ └── com/
│ │ │ │ └── alibaba/
│ │ │ │ └── datax/
│ │ │ │ └── example/
│ │ │ │ ├── ExampleContainer.java
│ │ │ │ ├── Main.java
│ │ │ │ └── util/
│ │ │ │ ├── ExampleConfigParser.java
│ │ │ │ └── PathUtil.java
│ │ │ └── resources/
│ │ │ └── example/
│ │ │ └── conf/
│ │ │ └── core.json
│ │ └── test/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── example/
│ │ │ └── util/
│ │ │ └── PathUtilTest.java
│ │ └── resources/
│ │ └── pathTest.json
│ ├── datax-example-neo4j/
│ │ ├── pom.xml
│ │ └── src/
│ │ └── test/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── example/
│ │ │ └── neo4j/
│ │ │ └── StreamReader2Neo4jWriterTest.java
│ │ └── resources/
│ │ └── streamreader2neo4j.json
│ ├── datax-example-streamreader/
│ │ ├── pom.xml
│ │ └── src/
│ │ └── test/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── example/
│ │ │ └── streamreader/
│ │ │ └── StreamReader2StreamWriterTest.java
│ │ └── resources/
│ │ └── stream2stream.json
│ ├── doc/
│ │ └── README.md
│ └── pom.xml
├── dataxPluginDev.md
├── dorisreader/
│ ├── doc/
│ │ └── dorisreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── dorisreader/
│ │ ├── DorisReader.java
│ │ └── DorisReaderErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── doriswriter/
│ ├── doc/
│ │ ├── doriswriter.md
│ │ └── mysql2doris.json
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── doriswriter/
│ │ ├── DelimiterParser.java
│ │ ├── DorisBaseCodec.java
│ │ ├── DorisCodec.java
│ │ ├── DorisCodecFactory.java
│ │ ├── DorisCsvCodec.java
│ │ ├── DorisJsonCodec.java
│ │ ├── DorisStreamLoadObserver.java
│ │ ├── DorisUtil.java
│ │ ├── DorisWriter.java
│ │ ├── DorisWriterExcetion.java
│ │ ├── DorisWriterManager.java
│ │ ├── Keys.java
│ │ └── WriterTuple.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── drdsreader/
│ ├── doc/
│ │ └── drdsreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── drdsreader/
│ │ ├── DrdsReader.java
│ │ ├── DrdsReaderErrorCode.java
│ │ └── DrdsReaderSplitUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── drdswriter/
│ ├── doc/
│ │ └── drdswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── drdswriter/
│ │ └── DrdsWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── elasticsearchwriter/
│ ├── README.md
│ ├── build.sh
│ ├── doc/
│ │ └── elasticsearchwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── elasticsearchwriter/
│ │ ├── ElasticSearchClient.java
│ │ ├── ElasticSearchColumn.java
│ │ ├── ElasticSearchFieldType.java
│ │ ├── ElasticSearchWriter.java
│ │ ├── ElasticSearchWriterErrorCode.java
│ │ ├── JsonPathUtil.java
│ │ ├── JsonUtil.java
│ │ ├── Key.java
│ │ ├── NoReRunException.java
│ │ ├── PartitionColumn.java
│ │ ├── PrimaryKeyInfo.java
│ │ └── jest/
│ │ ├── ClusterInfo.java
│ │ ├── ClusterInfoResult.java
│ │ └── PutMapping7.java
│ └── resources/
│ └── plugin.json
├── ftpreader/
│ ├── doc/
│ │ └── ftpreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── ftpreader/
│ │ ├── Constant.java
│ │ ├── FtpHelper.java
│ │ ├── FtpReader.java
│ │ ├── FtpReaderErrorCode.java
│ │ ├── Key.java
│ │ ├── SftpHelper.java
│ │ └── StandardFtpHelper.java
│ └── resources/
│ ├── plugin-template.json
│ ├── plugin.json
│ └── plugin_job_template.json
├── ftpwriter/
│ ├── doc/
│ │ ├── .gitkeep
│ │ └── ftpwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── ftpwriter/
│ │ ├── FtpWriter.java
│ │ ├── FtpWriterErrorCode.java
│ │ ├── Key.java
│ │ └── util/
│ │ ├── Constant.java
│ │ ├── IFtpHelper.java
│ │ ├── SftpHelperImpl.java
│ │ └── StandardFtpHelperImpl.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── gaussdbreader/
│ ├── doc/
│ │ └── gaussdbreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── gaussdbreader/
│ │ ├── Constant.java
│ │ └── GaussDbReader.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── gaussdbwriter/
│ ├── doc/
│ │ └── gaussdbwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── gaussdbwriter/
│ │ └── GaussDbWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── gdbreader/
│ ├── doc/
│ │ └── gdbreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── gdbreader/
│ │ ├── GdbReader.java
│ │ ├── GdbReaderErrorCode.java
│ │ ├── Key.java
│ │ ├── mapping/
│ │ │ ├── DefaultGdbMapper.java
│ │ │ ├── MappingRule.java
│ │ │ ├── MappingRuleFactory.java
│ │ │ └── ValueType.java
│ │ ├── model/
│ │ │ ├── AbstractGdbGraph.java
│ │ │ ├── GdbElement.java
│ │ │ ├── GdbGraph.java
│ │ │ └── ScriptGdbGraph.java
│ │ └── util/
│ │ └── ConfigHelper.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── gdbwriter/
│ ├── doc/
│ │ └── gdbwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── gdbwriter/
│ │ ├── GdbWriter.java
│ │ ├── GdbWriterErrorCode.java
│ │ ├── Key.java
│ │ ├── client/
│ │ │ ├── GdbGraphManager.java
│ │ │ └── GdbWriterConfig.java
│ │ ├── mapping/
│ │ │ ├── DefaultGdbMapper.java
│ │ │ ├── GdbMapper.java
│ │ │ ├── MapperConfig.java
│ │ │ ├── MappingRule.java
│ │ │ ├── MappingRuleFactory.java
│ │ │ └── ValueType.java
│ │ ├── model/
│ │ │ ├── AbstractGdbGraph.java
│ │ │ ├── GdbEdge.java
│ │ │ ├── GdbElement.java
│ │ │ ├── GdbGraph.java
│ │ │ ├── GdbVertex.java
│ │ │ └── ScriptGdbGraph.java
│ │ └── util/
│ │ ├── ConfigHelper.java
│ │ └── GdbDuplicateIdException.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hbase094xreader/
│ ├── doc/
│ │ ├── .gitkeep
│ │ └── hbase094xreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── hbase094xreader/
│ │ ├── ColumnType.java
│ │ ├── Constant.java
│ │ ├── Hbase094xHelper.java
│ │ ├── Hbase094xReader.java
│ │ ├── Hbase094xReaderErrorCode.java
│ │ ├── HbaseAbstractTask.java
│ │ ├── HbaseColumnCell.java
│ │ ├── Key.java
│ │ ├── ModeType.java
│ │ ├── MultiVersionFixedColumnTask.java
│ │ ├── MultiVersionTask.java
│ │ └── NormalTask.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hbase094xwriter/
│ ├── doc/
│ │ ├── .gitkeep
│ │ └── hbase094xwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── hbase094xwriter/
│ │ ├── ColumnType.java
│ │ ├── Constant.java
│ │ ├── Hbase094xHelper.java
│ │ ├── Hbase094xWriter.java
│ │ ├── Hbase094xWriterErrorCode.java
│ │ ├── HbaseAbstractTask.java
│ │ ├── Key.java
│ │ ├── ModeType.java
│ │ ├── NormalTask.java
│ │ └── NullModeType.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hbase11xreader/
│ ├── doc/
│ │ ├── .gitkeep
│ │ └── hbase11xreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── hbase11xreader/
│ │ ├── ColumnType.java
│ │ ├── Constant.java
│ │ ├── Hbase11xHelper.java
│ │ ├── Hbase11xReader.java
│ │ ├── Hbase11xReaderErrorCode.java
│ │ ├── HbaseAbstractTask.java
│ │ ├── HbaseColumnCell.java
│ │ ├── Key.java
│ │ ├── ModeType.java
│ │ ├── MultiVersionDynamicColumnTask.java
│ │ ├── MultiVersionFixedColumnTask.java
│ │ ├── MultiVersionTask.java
│ │ └── NormalTask.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hbase11xsqlreader/
│ ├── doc/
│ │ └── hbase11xsqlreader.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ └── hbase11xsqlreader/
│ │ │ ├── HadoopSerializationUtil.java
│ │ │ ├── HbaseSQLHelper.java
│ │ │ ├── HbaseSQLReader.java
│ │ │ ├── HbaseSQLReaderConfig.java
│ │ │ ├── HbaseSQLReaderErrorCode.java
│ │ │ ├── HbaseSQLReaderTask.java
│ │ │ ├── Key.java
│ │ │ ├── LocalStrings.properties
│ │ │ ├── LocalStrings_en_US.properties
│ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ └── LocalStrings_zh_CN.properties
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── reader/
│ └── hbase11xsqlreader/
│ ├── HbaseSQLHelperTest.java
│ └── HbaseSQLReaderTaskTest.java
├── hbase11xsqlwriter/
│ ├── doc/
│ │ └── hbase11xsqlwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── hbase11xsqlwriter/
│ │ ├── Constant.java
│ │ ├── HbaseSQLHelper.java
│ │ ├── HbaseSQLWriter.java
│ │ ├── HbaseSQLWriterConfig.java
│ │ ├── HbaseSQLWriterErrorCode.java
│ │ ├── HbaseSQLWriterTask.java
│ │ ├── Key.java
│ │ ├── NullModeType.java
│ │ └── ThinClientPTable.java
│ └── resources/
│ └── plugin.json
├── hbase11xwriter/
│ ├── doc/
│ │ ├── .gitkeep
│ │ └── hbase11xwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── hbase11xwriter/
│ │ ├── ColumnType.java
│ │ ├── Constant.java
│ │ ├── Hbase11xHelper.java
│ │ ├── Hbase11xWriter.java
│ │ ├── Hbase11xWriterErrorCode.java
│ │ ├── HbaseAbstractTask.java
│ │ ├── Key.java
│ │ ├── ModeType.java
│ │ ├── MultiVersionTask.java
│ │ ├── NormalTask.java
│ │ └── NullModeType.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hbase20xsqlreader/
│ ├── doc/
│ │ └── hbase20xsqlreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── hbase20xsqlreader/
│ │ ├── Constant.java
│ │ ├── HBase20SQLReaderHelper.java
│ │ ├── HBase20xSQLReader.java
│ │ ├── HBase20xSQLReaderErrorCode.java
│ │ ├── HBase20xSQLReaderTask.java
│ │ └── Key.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hbase20xsqlwriter/
│ ├── doc/
│ │ └── hbase20xsqlwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── hbase20xsqlwriter/
│ │ ├── Constant.java
│ │ ├── HBase20xSQLHelper.java
│ │ ├── HBase20xSQLWriter.java
│ │ ├── HBase20xSQLWriterErrorCode.java
│ │ ├── HBase20xSQLWriterTask.java
│ │ ├── Key.java
│ │ └── NullModeType.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hdfsreader/
│ ├── doc/
│ │ └── hdfsreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── hdfsreader/
│ │ ├── Constant.java
│ │ ├── DFSUtil.java
│ │ ├── HdfsFileType.java
│ │ ├── HdfsPathFilter.java
│ │ ├── HdfsReader.java
│ │ ├── HdfsReaderErrorCode.java
│ │ ├── Key.java
│ │ ├── ParquetMessageHelper.java
│ │ └── ParquetMeta.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hdfswriter/
│ ├── doc/
│ │ └── hdfswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── hdfswriter/
│ │ ├── Constant.java
│ │ ├── HdfsHelper.java
│ │ ├── HdfsWriter.java
│ │ ├── HdfsWriterErrorCode.java
│ │ ├── Key.java
│ │ ├── ParquetFileProccessor.java
│ │ ├── ParquetFileSupport.java
│ │ └── SupportHiveDataType.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── hologresjdbcwriter/
│ ├── doc/
│ │ └── hologresjdbcwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── hologresjdbcwriter/
│ │ ├── BaseWriter.java
│ │ ├── Constant.java
│ │ ├── HologresJdbcWriter.java
│ │ ├── Key.java
│ │ └── util/
│ │ ├── ConfLoader.java
│ │ ├── OriginalConfPretreatmentUtil.java
│ │ └── WriterUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── introduction.md
├── kingbaseesreader/
│ ├── doc/
│ │ └── kingbaseesreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── kingbaseesreader/
│ │ ├── Constant.java
│ │ └── KingbaseesReader.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── kingbaseeswriter/
│ ├── doc/
│ │ └── kingbaseeswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── kingbaseeswriter/
│ │ └── KingbaseesWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── kuduwriter/
│ ├── README.md
│ ├── doc/
│ │ └── kuduwirter.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── q1/
│ │ │ ├── datax/
│ │ │ │ └── plugin/
│ │ │ │ └── writer/
│ │ │ │ └── kudu11xwriter/
│ │ │ │ ├── ColumnType.java
│ │ │ │ ├── Constant.java
│ │ │ │ ├── InsertModeType.java
│ │ │ │ ├── Key.java
│ │ │ │ ├── Kudu11xHelper.java
│ │ │ │ ├── Kudu11xWriter.java
│ │ │ │ ├── Kudu11xWriterErrorcode.java
│ │ │ │ └── KuduWriterTask.java
│ │ │ └── kudu/
│ │ │ └── conf/
│ │ │ └── KuduConfig.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ └── java/
│ └── com/
│ └── dai/
│ └── test.java
├── license.txt
├── loghubreader/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── loghubreader/
│ │ ├── Constant.java
│ │ ├── Key.java
│ │ ├── LogHubReader.java
│ │ └── LogHubReaderErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── loghubwriter/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── loghubwriter/
│ │ ├── Key.java
│ │ ├── LogHubWriter.java
│ │ └── LogHubWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── milvuswriter/
│ ├── doc/
│ │ └── milvuswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── milvuswriter/
│ │ ├── KeyConstant.java
│ │ ├── MilvusBufferWriter.java
│ │ ├── MilvusClient.java
│ │ ├── MilvusColumn.java
│ │ ├── MilvusCreateCollection.java
│ │ ├── MilvusWriter.java
│ │ ├── MilvusWriterErrorCode.java
│ │ └── enums/
│ │ ├── SchemaCreateModeEnum.java
│ │ └── WriteModeEnum.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── mongodbreader/
│ ├── doc/
│ │ └── mongodbreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── mongodbreader/
│ │ ├── KeyConstant.java
│ │ ├── MongoDBReader.java
│ │ ├── MongoDBReaderErrorCode.java
│ │ └── util/
│ │ ├── CollectionSplitUtil.java
│ │ └── MongoUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── mongodbwriter/
│ ├── doc/
│ │ └── mongodbwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── mongodbwriter/
│ │ ├── KeyConstant.java
│ │ ├── MongoDBWriter.java
│ │ ├── MongoDBWriterErrorCode.java
│ │ └── util/
│ │ └── MongoUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── mysqlreader/
│ ├── doc/
│ │ └── mysqlreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── mysqlreader/
│ │ ├── MysqlReader.java
│ │ └── MysqlReaderErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── mysqlwriter/
│ ├── doc/
│ │ └── mysqlwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── mysqlwriter/
│ │ └── MysqlWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── neo4jwriter/
│ ├── doc/
│ │ └── neo4jwriter.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── writer/
│ │ │ └── neo4jwriter/
│ │ │ ├── Neo4jClient.java
│ │ │ ├── Neo4jWriter.java
│ │ │ ├── adapter/
│ │ │ │ ├── DateAdapter.java
│ │ │ │ └── ValueAdapter.java
│ │ │ ├── config/
│ │ │ │ ├── ConfigConstants.java
│ │ │ │ ├── Neo4jProperty.java
│ │ │ │ └── Option.java
│ │ │ ├── element/
│ │ │ │ └── PropertyType.java
│ │ │ └── exception/
│ │ │ └── Neo4jErrorCode.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ ├── Neo4jWriterTest.java
│ │ └── mock/
│ │ ├── MockRecord.java
│ │ └── MockUtil.java
│ └── resources/
│ ├── allTypeFieldNode.json
│ ├── dynamicLabel.json
│ ├── relationship.json
│ └── streamreader2neo4j.json
├── obhbasereader/
│ ├── doc/
│ │ └── obhbasereader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── obhbasereader/
│ │ ├── Constant.java
│ │ ├── HTableManager.java
│ │ ├── HbaseColumnCell.java
│ │ ├── HbaseReaderErrorCode.java
│ │ ├── Key.java
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ ├── LocalStrings_zh_TW.properties
│ │ ├── ObHbaseReader.java
│ │ ├── enums/
│ │ │ ├── ColumnType.java
│ │ │ ├── FetchVersion.java
│ │ │ └── ModeType.java
│ │ ├── ext/
│ │ │ └── ServerConnectInfo.java
│ │ ├── task/
│ │ │ ├── AbstractHbaseTask.java
│ │ │ ├── AbstractScanReader.java
│ │ │ ├── SQLNormalModeReader.java
│ │ │ ├── ScanMultiVersionReader.java
│ │ │ └── ScanNormalModeReader.java
│ │ └── util/
│ │ ├── HbaseSplitUtil.java
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ ├── LocalStrings_zh_TW.properties
│ │ ├── ObHbaseReaderUtil.java
│ │ └── SqlReaderSplitUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── obhbasewriter/
│ ├── doc/
│ │ └── obhbasewriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── obhbasewriter/
│ │ ├── ColumnType.java
│ │ ├── Config.java
│ │ ├── ConfigKey.java
│ │ ├── ConfigValidator.java
│ │ ├── Constant.java
│ │ ├── Hbase094xWriterErrorCode.java
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ ├── LocalStrings_zh_TW.properties
│ │ ├── ModeType.java
│ │ ├── NullModeType.java
│ │ ├── ObHTableInfo.java
│ │ ├── ObHbaseWriter.java
│ │ ├── ext/
│ │ │ ├── LocalStrings.properties
│ │ │ ├── LocalStrings_en_US.properties
│ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ ├── ObDataSourceErrorCode.java
│ │ │ ├── ObHbaseTableHolder.java
│ │ │ └── ServerConnectInfo.java
│ │ ├── task/
│ │ │ ├── LocalStrings.properties
│ │ │ ├── LocalStrings_en_US.properties
│ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ ├── MultiVersionWriteTask.java
│ │ │ ├── NormalWriteTask.java
│ │ │ ├── ObHBaseWriteTask.java
│ │ │ └── PutTask.java
│ │ └── util/
│ │ ├── LocalStrings.properties
│ │ ├── LocalStrings_en_US.properties
│ │ ├── LocalStrings_ja_JP.properties
│ │ ├── LocalStrings_zh_CN.properties
│ │ ├── LocalStrings_zh_HK.properties
│ │ ├── LocalStrings_zh_TW.properties
│ │ └── ObHbaseWriterUtils.java
│ └── resources/
│ └── plugin.json
├── oceanbasev10reader/
│ ├── doc/
│ │ └── oceanbasev10reader.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ └── oceanbasev10reader/
│ │ │ ├── Config.java
│ │ │ ├── OceanBaseReader.java
│ │ │ ├── ext/
│ │ │ │ ├── Constant.java
│ │ │ │ ├── ObReaderKey.java
│ │ │ │ ├── ReaderJob.java
│ │ │ │ └── ReaderTask.java
│ │ │ └── util/
│ │ │ ├── ExecutorTemplate.java
│ │ │ ├── ObReaderUtils.java
│ │ │ ├── PartInfo.java
│ │ │ ├── PartType.java
│ │ │ ├── PartitionSplitUtil.java
│ │ │ └── TaskContext.java
│ │ └── resources/
│ │ └── plugin.json
│ └── test/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── reader/
│ └── oceanbasev10reader/
│ └── util/
│ └── ObReaderUtilsTest.java
├── oceanbasev10writer/
│ ├── doc/
│ │ └── oceanbasev10writer.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── oceanbasev10writer/
│ │ ├── Config.java
│ │ ├── OceanBaseV10Writer.java
│ │ ├── common/
│ │ │ ├── Table.java
│ │ │ └── TableCache.java
│ │ ├── directPath/
│ │ │ ├── AbstractRestrictedConnection.java
│ │ │ ├── AbstractRestrictedPreparedStatement.java
│ │ │ ├── DirectLoaderBuilder.java
│ │ │ ├── DirectPathConnection.java
│ │ │ ├── DirectPathConstants.java
│ │ │ ├── DirectPathPreparedStatement.java
│ │ │ └── ObTableDirectLoad.java
│ │ ├── ext/
│ │ │ ├── AbstractConnHolder.java
│ │ │ ├── ConnHolder.java
│ │ │ ├── DataBaseWriterBuffer.java
│ │ │ ├── DirectPathAbstractConnHolder.java
│ │ │ ├── DirectPathConnHolder.java
│ │ │ ├── OBDataSourceV10.java
│ │ │ ├── OCJConnHolder.java
│ │ │ ├── ObClientConnHolder.java
│ │ │ ├── ObDataSourceErrorCode.java
│ │ │ └── ServerConnectInfo.java
│ │ ├── part/
│ │ │ ├── IObPartCalculator.java
│ │ │ ├── ObPartitionCalculatorV1.java
│ │ │ └── ObPartitionCalculatorV2.java
│ │ ├── task/
│ │ │ ├── AbstractInsertTask.java
│ │ │ ├── ColumnMetaCache.java
│ │ │ ├── ConcurrentTableWriterTask.java
│ │ │ ├── DirectPathInsertTask.java
│ │ │ ├── InsertTask.java
│ │ │ ├── SingleTableWriterTask.java
│ │ │ └── WriterThreadPool.java
│ │ └── util/
│ │ ├── DbUtils.java
│ │ └── ObWriterUtils.java
│ └── resources/
│ └── plugin.json
├── ocswriter/
│ ├── doc/
│ │ └── ocswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── ocswriter/
│ │ ├── Key.java
│ │ ├── OcsWriter.java
│ │ └── utils/
│ │ ├── CommonUtils.java
│ │ ├── ConfigurationChecker.java
│ │ └── OcsWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── odpsreader/
│ ├── doc/
│ │ └── odpsreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── odpsreader/
│ │ ├── ColumnType.java
│ │ ├── Constant.java
│ │ ├── InternalColumnInfo.java
│ │ ├── Key.java
│ │ ├── LocalStrings.properties
│ │ ├── OdpsReader.java
│ │ ├── OdpsReaderErrorCode.java
│ │ ├── ReaderProxy.java
│ │ └── util/
│ │ ├── LocalStrings.properties
│ │ ├── OdpsExceptionMsg.java
│ │ ├── OdpsSplitUtil.java
│ │ ├── OdpsUtil.java
│ │ ├── SqliteUtil.java
│ │ └── UserConfiguredPartitionClassification.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── odpswriter/
│ ├── doc/
│ │ └── odpswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── odpswriter/
│ │ ├── Constant.java
│ │ ├── DateTransForm.java
│ │ ├── Key.java
│ │ ├── LocalStrings.properties
│ │ ├── OdpsWriter.java
│ │ ├── OdpsWriterErrorCode.java
│ │ ├── OdpsWriterProxy.java
│ │ ├── model/
│ │ │ ├── PartitionInfo.java
│ │ │ ├── UserDefinedFunction.java
│ │ │ └── UserDefinedFunctionRule.java
│ │ └── util/
│ │ ├── CustomPartitionUtils.java
│ │ ├── LocalStrings.properties
│ │ ├── OdpsExceptionMsg.java
│ │ └── OdpsUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── opentsdbreader/
│ ├── doc/
│ │ └── opentsdbreader.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ ├── conn/
│ │ │ │ ├── CliQuery.java
│ │ │ │ ├── Connection4TSDB.java
│ │ │ │ ├── DataPoint4TSDB.java
│ │ │ │ ├── DumpSeries.java
│ │ │ │ ├── OpenTSDBConnection.java
│ │ │ │ └── OpenTSDBDump.java
│ │ │ ├── opentsdbreader/
│ │ │ │ ├── Constant.java
│ │ │ │ ├── Key.java
│ │ │ │ ├── OpenTSDBReader.java
│ │ │ │ └── OpenTSDBReaderErrorCode.java
│ │ │ └── util/
│ │ │ ├── HttpUtils.java
│ │ │ ├── TSDBUtils.java
│ │ │ └── TimeUtils.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── reader/
│ ├── conn/
│ │ └── OpenTSDBConnectionTest.java
│ └── util/
│ ├── Const.java
│ ├── HttpUtilsTest.java
│ ├── TSDBTest.java
│ └── TimeUtilsTest.java
├── oraclereader/
│ ├── doc/
│ │ └── oraclereader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── oraclereader/
│ │ ├── Constant.java
│ │ ├── OracleReader.java
│ │ └── OracleReaderErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── oraclewriter/
│ ├── doc/
│ │ └── oraclewriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── oraclewriter/
│ │ ├── OracleWriter.java
│ │ └── OracleWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── oscarwriter/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── oscarwriter/
│ │ ├── OscarWriter.java
│ │ └── OscarWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── ossreader/
│ ├── doc/
│ │ └── ossreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── ossreader/
│ │ ├── Constant.java
│ │ ├── Key.java
│ │ ├── OssInputStream.java
│ │ ├── OssReader.java
│ │ ├── OssReaderErrorCode.java
│ │ └── util/
│ │ ├── HdfsParquetUtil.java
│ │ ├── OssSplitUtil.java
│ │ └── OssUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── osswriter/
│ ├── doc/
│ │ └── osswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── osswriter/
│ │ ├── Constant.java
│ │ ├── Key.java
│ │ ├── OssSingleObject.java
│ │ ├── OssWriter.java
│ │ ├── OssWriterErrorCode.java
│ │ ├── OssWriterProxy.java
│ │ ├── parquet/
│ │ │ ├── ParquetFileProccessor.java
│ │ │ └── ParquetFileSupport.java
│ │ └── util/
│ │ ├── HandlerUtil.java
│ │ ├── HdfsParquetUtil.java
│ │ └── OssUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── otsreader/
│ ├── doc/
│ │ └── otsreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── otsreader/
│ │ ├── IOtsReaderMasterProxy.java
│ │ ├── IOtsReaderSlaveProxy.java
│ │ ├── OtsReader.java
│ │ ├── OtsReaderMasterProxy.java
│ │ ├── OtsReaderSlaveMetaProxy.java
│ │ ├── OtsReaderSlaveMultiVersionProxy.java
│ │ ├── OtsReaderSlaveNormalProxy.java
│ │ ├── OtsReaderSlaveProxyOld.java
│ │ ├── adaptor/
│ │ │ ├── ColumnAdaptor.java
│ │ │ └── PrimaryKeyValueAdaptor.java
│ │ ├── callable/
│ │ │ ├── GetFirstRowPrimaryKeyCallable.java
│ │ │ ├── GetRangeCallable.java
│ │ │ ├── GetRangeCallableOld.java
│ │ │ ├── GetTableMetaCallable.java
│ │ │ ├── GetTimeseriesSplitCallable.java
│ │ │ └── ScanTimeseriesDataCallable.java
│ │ ├── model/
│ │ │ ├── DefaultNoRetry.java
│ │ │ ├── OTSColumn.java
│ │ │ ├── OTSConf.java
│ │ │ ├── OTSConst.java
│ │ │ ├── OTSCriticalException.java
│ │ │ ├── OTSErrorCode.java
│ │ │ ├── OTSMode.java
│ │ │ ├── OTSMultiVersionConf.java
│ │ │ ├── OTSPrimaryKeyColumn.java
│ │ │ └── OTSRange.java
│ │ └── utils/
│ │ ├── Common.java
│ │ ├── CommonOld.java
│ │ ├── CompareHelper.java
│ │ ├── Constant.java
│ │ ├── DefaultNoRetry.java
│ │ ├── GsonParser.java
│ │ ├── Key.java
│ │ ├── OtsHelper.java
│ │ ├── OtsReaderError.java
│ │ ├── ParamChecker.java
│ │ ├── ParamCheckerOld.java
│ │ ├── ParamParser.java
│ │ ├── RangeSplit.java
│ │ ├── ReaderModelParser.java
│ │ ├── RetryHelper.java
│ │ ├── RetryHelperOld.java
│ │ └── TranformHelper.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── otsstreamreader/
│ ├── README.md
│ ├── pom.xml
│ ├── src/
│ │ └── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ └── otsstreamreader/
│ │ │ └── internal/
│ │ │ ├── LocalStrings.properties
│ │ │ ├── LocalStrings_en_US.properties
│ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ ├── OTSReaderError.java
│ │ │ ├── OTSStreamReader.java
│ │ │ ├── OTSStreamReaderException.java
│ │ │ ├── OTSStreamReaderMasterProxy.java
│ │ │ ├── OTSStreamReaderSlaveProxy.java
│ │ │ ├── config/
│ │ │ │ ├── LocalStrings.properties
│ │ │ │ ├── LocalStrings_en_US.properties
│ │ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ │ ├── Mode.java
│ │ │ │ ├── OTSRetryStrategyForStreamReader.java
│ │ │ │ ├── OTSStreamReaderConfig.java
│ │ │ │ ├── OTSStreamReaderConstants.java
│ │ │ │ └── StatusTableConstants.java
│ │ │ ├── core/
│ │ │ │ ├── CheckpointTimeTracker.java
│ │ │ │ ├── IStreamRecordSender.java
│ │ │ │ ├── LocalStrings.properties
│ │ │ │ ├── LocalStrings_en_US.properties
│ │ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ │ ├── MultiVerModeRecordSender.java
│ │ │ │ ├── OTSStreamReaderChecker.java
│ │ │ │ ├── RecordProcessor.java
│ │ │ │ ├── ShardStatusChecker.java
│ │ │ │ └── SingleVerAndUpOnlyModeRecordSender.java
│ │ │ ├── model/
│ │ │ │ ├── LocalStrings.properties
│ │ │ │ ├── LocalStrings_en_US.properties
│ │ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ │ ├── OTSErrorCode.java
│ │ │ │ ├── OTSStreamJobShard.java
│ │ │ │ ├── ShardCheckpoint.java
│ │ │ │ └── StreamJob.java
│ │ │ └── utils/
│ │ │ ├── ColumnValueTransformHelper.java
│ │ │ ├── GsonParser.java
│ │ │ ├── LocalStrings.properties
│ │ │ ├── LocalStrings_en_US.properties
│ │ │ ├── LocalStrings_ja_JP.properties
│ │ │ ├── LocalStrings_zh_CN.properties
│ │ │ ├── LocalStrings_zh_HK.properties
│ │ │ ├── LocalStrings_zh_TW.properties
│ │ │ ├── OTSHelper.java
│ │ │ ├── OTSStreamJobShardUtil.java
│ │ │ ├── ParamChecker.java
│ │ │ └── TimeUtils.java
│ │ └── resources/
│ │ ├── log4j2.xml
│ │ └── plugin.json
│ └── tools/
│ ├── config.json
│ ├── tablestore_streamreader_console.py
│ └── tabulate.py
├── otswriter/
│ ├── doc/
│ │ └── otswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── otswriter/
│ │ ├── IOtsWriterMasterProxy.java
│ │ ├── IOtsWriterSlaveProxy.java
│ │ ├── Key.java
│ │ ├── OTSCriticalException.java
│ │ ├── OTSErrorCode.java
│ │ ├── OtsWriter.java
│ │ ├── OtsWriterError.java
│ │ ├── OtsWriterMasterProxy.java
│ │ ├── OtsWriterSlaveProxyMultiversion.java
│ │ ├── OtsWriterSlaveProxyNormal.java
│ │ ├── OtsWriterSlaveProxyOld.java
│ │ ├── callable/
│ │ │ ├── BatchWriteRowCallable.java
│ │ │ ├── GetTableMetaCallable.java
│ │ │ ├── GetTableMetaCallableOld.java
│ │ │ ├── PutRowChangeCallable.java
│ │ │ ├── PutTimeseriesDataCallable.java
│ │ │ └── UpdateRowChangeCallable.java
│ │ ├── model/
│ │ │ ├── OTSAttrColumn.java
│ │ │ ├── OTSBatchWriteRowTaskManager.java
│ │ │ ├── OTSBatchWriterRowTask.java
│ │ │ ├── OTSBlockingExecutor.java
│ │ │ ├── OTSConf.java
│ │ │ ├── OTSConst.java
│ │ │ ├── OTSErrorMessage.java
│ │ │ ├── OTSLine.java
│ │ │ ├── OTSMode.java
│ │ │ ├── OTSOpType.java
│ │ │ ├── OTSSendBuffer.java
│ │ │ ├── OTSTaskManagerInterface.java
│ │ │ ├── OTSTimeseriesRowTask.java
│ │ │ ├── OTSTimeseriesRowTaskManager.java
│ │ │ ├── RowDeleteChangeWithRecord.java
│ │ │ ├── RowPutChangeWithRecord.java
│ │ │ └── RowUpdateChangeWithRecord.java
│ │ └── utils/
│ │ ├── CalculateHelper.java
│ │ ├── CollectorUtil.java
│ │ ├── ColumnConversion.java
│ │ ├── ColumnConversionOld.java
│ │ ├── Common.java
│ │ ├── CommonOld.java
│ │ ├── DefaultNoRetry.java
│ │ ├── GsonParser.java
│ │ ├── LineAndError.java
│ │ ├── ParamChecker.java
│ │ ├── ParseRecord.java
│ │ ├── RetryHelper.java
│ │ ├── WithRecord.java
│ │ ├── WriterModelParser.java
│ │ └── WriterRetryPolicy.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── package.xml
├── plugin-rdbms-util/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── rdbms/
│ ├── reader/
│ │ ├── CommonRdbmsReader.java
│ │ ├── Constant.java
│ │ ├── Key.java
│ │ ├── ResultSetReadProxy.java
│ │ └── util/
│ │ ├── HintUtil.java
│ │ ├── ObVersion.java
│ │ ├── OriginalConfPretreatmentUtil.java
│ │ ├── PreCheckTask.java
│ │ ├── ReaderSplitUtil.java
│ │ └── SingleTableSplitUtil.java
│ ├── util/
│ │ ├── ConnectionFactory.java
│ │ ├── Constant.java
│ │ ├── DBUtil.java
│ │ ├── DBUtilErrorCode.java
│ │ ├── DataBaseType.java
│ │ ├── JdbcConnectionFactory.java
│ │ ├── RdbmsException.java
│ │ ├── RdbmsRangeSplitWrap.java
│ │ ├── SplitedSlice.java
│ │ └── TableExpandUtil.java
│ └── writer/
│ ├── CommonRdbmsWriter.java
│ ├── Constant.java
│ ├── Key.java
│ ├── MysqlWriterErrorCode.java
│ └── util/
│ ├── OriginalConfPretreatmentUtil.java
│ └── WriterUtil.java
├── plugin-unstructured-storage-util/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── unstructuredstorage/
│ ├── FileFormat.java
│ ├── LocalStrings.properties
│ ├── LocalStrings_en_US.properties
│ ├── LocalStrings_ja_JP.properties
│ ├── LocalStrings_zh_CN.properties
│ ├── LocalStrings_zh_HK.properties
│ ├── LocalStrings_zh_TW.properties
│ ├── reader/
│ │ ├── ColumnEntry.java
│ │ ├── Constant.java
│ │ ├── ExpandLzopInputStream.java
│ │ ├── Key.java
│ │ ├── UnstructuredStorageReaderErrorCode.java
│ │ ├── UnstructuredStorageReaderUtil.java
│ │ ├── ZipCycleInputStream.java
│ │ ├── binaryFileUtil/
│ │ │ ├── BinaryFileReaderUtil.java
│ │ │ └── ByteUtils.java
│ │ └── split/
│ │ ├── StartEndPair.java
│ │ └── UnstructuredSplitUtil.java
│ ├── util/
│ │ ├── ColumnTypeUtil.java
│ │ └── HdfsUtil.java
│ └── writer/
│ ├── Constant.java
│ ├── DataXCsvWriter.java
│ ├── Key.java
│ ├── SqlWriter.java
│ ├── TextCsvWriterManager.java
│ ├── UnstructuredStorageWriterErrorCode.java
│ ├── UnstructuredStorageWriterUtil.java
│ ├── UnstructuredWriter.java
│ └── binaryFileUtil/
│ ├── BinaryFileWriterErrorCode.java
│ └── BinaryFileWriterUtil.java
├── pom.xml
├── postgresqlreader/
│ ├── doc/
│ │ └── postgresqlreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── postgresqlreader/
│ │ ├── Constant.java
│ │ └── PostgresqlReader.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── postgresqlwriter/
│ ├── doc/
│ │ └── postgresqlwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── postgresqlwriter/
│ │ └── PostgresqlWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── rdbmsreader/
│ ├── doc/
│ │ └── rdbmsreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── rdbmsreader/
│ │ ├── Constant.java
│ │ ├── RdbmsReader.java
│ │ └── SubCommonRdbmsReader.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── rdbmswriter/
│ ├── doc/
│ │ └── rdbmswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── rdbmswriter/
│ │ ├── RdbmsWriter.java
│ │ └── SubCommonRdbmsWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── rpm/
│ ├── t_dp_dw_datax_3_core_all-build.sh
│ └── t_dp_dw_datax_3_hook_dqc-build.sh
├── selectdbwriter/
│ ├── doc/
│ │ ├── selectdbwriter.md
│ │ └── stream2selectdb.json
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── selectdbwriter/
│ │ ├── BaseResponse.java
│ │ ├── CopyIntoResp.java
│ │ ├── CopySQLBuilder.java
│ │ ├── DelimiterParser.java
│ │ ├── HttpPostBuilder.java
│ │ ├── HttpPutBuilder.java
│ │ ├── Keys.java
│ │ ├── SelectdbBaseCodec.java
│ │ ├── SelectdbCodec.java
│ │ ├── SelectdbCodecFactory.java
│ │ ├── SelectdbCopyIntoObserver.java
│ │ ├── SelectdbCsvCodec.java
│ │ ├── SelectdbJsonCodec.java
│ │ ├── SelectdbUtil.java
│ │ ├── SelectdbWriter.java
│ │ ├── SelectdbWriterException.java
│ │ ├── SelectdbWriterManager.java
│ │ └── WriterTuple.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── sqlserverreader/
│ ├── doc/
│ │ └── sqlserverreader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── sqlserverreader/
│ │ ├── Constant.java
│ │ ├── Key.java
│ │ ├── SqlServerReader.java
│ │ └── SqlServerReaderErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── sqlserverwriter/
│ ├── doc/
│ │ └── sqlserverwriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── sqlserverwriter/
│ │ ├── SqlServerWriter.java
│ │ └── SqlServerWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── starrocksreader/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── starrocksreader/
│ │ └── StarRocksReader.java
│ └── resources/
│ └── plugin.json
├── starrockswriter/
│ ├── doc/
│ │ └── starrockswriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── starrocks/
│ │ └── connector/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── starrockswriter/
│ │ ├── StarRocksWriter.java
│ │ ├── StarRocksWriterOptions.java
│ │ ├── manager/
│ │ │ ├── StarRocksFlushTuple.java
│ │ │ ├── StarRocksStreamLoadFailedException.java
│ │ │ ├── StarRocksStreamLoadVisitor.java
│ │ │ └── StarRocksWriterManager.java
│ │ ├── row/
│ │ │ ├── StarRocksBaseSerializer.java
│ │ │ ├── StarRocksCsvSerializer.java
│ │ │ ├── StarRocksDelimiterParser.java
│ │ │ ├── StarRocksISerializer.java
│ │ │ ├── StarRocksJsonSerializer.java
│ │ │ └── StarRocksSerializerFactory.java
│ │ └── util/
│ │ └── StarRocksWriterUtil.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── streamreader/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── streamreader/
│ │ ├── Constant.java
│ │ ├── Key.java
│ │ ├── StreamReader.java
│ │ └── StreamReaderErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── streamwriter/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── streamwriter/
│ │ ├── Key.java
│ │ ├── StreamWriter.java
│ │ └── StreamWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── sybasereader/
│ ├── doc/
│ │ └── sybasereader.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ └── sybasereader/
│ │ │ ├── Constants.java
│ │ │ └── SybaseReader.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── reader/
│ └── sybasereader/
│ └── SybaseDatabaseUnitTest.java
├── sybasewriter/
│ ├── doc/
│ │ └── sybasewriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ └── java/
│ ├── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── sybasewriter/
│ │ └── SybaseWriter.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── tdenginereader/
│ ├── doc/
│ │ └── tdenginereader-CN.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ ├── TDengineReader.java
│ │ │ └── TDengineReaderErrorCode.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ ├── TDengine2DMTest.java
│ │ ├── TDengine2StreamTest.java
│ │ └── TDengineReaderTest.java
│ └── resources/
│ ├── t2dm.json
│ ├── t2stream-1.json
│ └── t2stream-2.json
├── tdenginewriter/
│ ├── doc/
│ │ ├── tdenginewriter-CN.md
│ │ └── tdenginewriter.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── writer/
│ │ │ └── tdenginewriter/
│ │ │ ├── ColumnMeta.java
│ │ │ ├── Constants.java
│ │ │ ├── DataHandler.java
│ │ │ ├── DefaultDataHandler.java
│ │ │ ├── Key.java
│ │ │ ├── OpentsdbDataHandler.java
│ │ │ ├── SchemaManager.java
│ │ │ ├── TDengineWriter.java
│ │ │ ├── TDengineWriterErrorCode.java
│ │ │ ├── TableMeta.java
│ │ │ ├── TableType.java
│ │ │ └── TimestampPrecision.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── tdenginewriter/
│ │ ├── Csv2TDengineTest.java
│ │ ├── DM2TDengineTest.java
│ │ ├── DefaultDataHandlerTest.java
│ │ ├── Mongo2TDengineTest.java
│ │ ├── Mysql2TDengineTest.java
│ │ ├── Opentsdb2TDengineTest.java
│ │ ├── SchemaManagerTest.java
│ │ ├── Stream2TDengineTest.java
│ │ ├── TDengine2TDengineTest.java
│ │ └── TDengineWriterTest.java
│ └── resources/
│ ├── csv2t.json
│ ├── defaultJob.json
│ ├── dm-schema.sql
│ ├── dm2t-1.json
│ ├── dm2t-2.json
│ ├── dm2t-3.json
│ ├── dm2t-4.json
│ ├── incremental_sync/
│ │ ├── clean_env.sh
│ │ ├── csv2t-jni.json
│ │ ├── csv2t-restful.json
│ │ ├── dm2t-jni.json
│ │ ├── dm2t-restful.json
│ │ ├── dm2t-update.json
│ │ ├── dm2t_sync.sh
│ │ ├── t2dm-jni.json
│ │ ├── t2dm-restful.json
│ │ └── upload.sh
│ ├── m2t-1.json
│ ├── mongo2t.json
│ ├── o2t-1.json
│ ├── t2t-1.json
│ ├── t2t-2.json
│ ├── t2t-3.json
│ ├── t2t-4.json
│ └── weather.csv
├── transformer/
│ ├── doc/
│ │ ├── .gitkeep
│ │ └── transformer.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── transformer/
│ ├── ComplexTransformer.java
│ └── Transformer.java
├── tsdbreader/
│ ├── doc/
│ │ └── tsdbreader.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── reader/
│ │ │ └── tsdbreader/
│ │ │ ├── Constant.java
│ │ │ ├── Key.java
│ │ │ ├── TSDBReader.java
│ │ │ ├── TSDBReaderErrorCode.java
│ │ │ ├── conn/
│ │ │ │ ├── Connection4TSDB.java
│ │ │ │ ├── DataPoint4MultiFieldsTSDB.java
│ │ │ │ ├── DataPoint4TSDB.java
│ │ │ │ ├── MultiFieldQueryResult.java
│ │ │ │ ├── QueryResult.java
│ │ │ │ ├── TSDBConnection.java
│ │ │ │ └── TSDBDump.java
│ │ │ └── util/
│ │ │ ├── HttpUtils.java
│ │ │ ├── TSDBUtils.java
│ │ │ └── TimeUtils.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── reader/
│ └── tsdbreader/
│ ├── conn/
│ │ └── TSDBConnectionTest.java
│ └── util/
│ ├── Const.java
│ └── TimeUtilsTest.java
├── tsdbwriter/
│ ├── doc/
│ │ └── tsdbhttpwriter.md
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── package.xml
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── alibaba/
│ │ │ └── datax/
│ │ │ └── plugin/
│ │ │ └── writer/
│ │ │ ├── conn/
│ │ │ │ ├── Connection4TSDB.java
│ │ │ │ ├── DataPoint4TSDB.java
│ │ │ │ └── TSDBConnection.java
│ │ │ ├── tsdbwriter/
│ │ │ │ ├── Constant.java
│ │ │ │ ├── Key.java
│ │ │ │ ├── SourceDBType.java
│ │ │ │ ├── TSDBConverter.java
│ │ │ │ ├── TSDBModel.java
│ │ │ │ ├── TSDBWriter.java
│ │ │ │ └── TSDBWriterErrorCode.java
│ │ │ └── util/
│ │ │ ├── HttpUtils.java
│ │ │ └── TSDBUtils.java
│ │ └── resources/
│ │ ├── plugin.json
│ │ └── plugin_job_template.json
│ └── test/
│ └── java/
│ └── com/
│ └── alibaba/
│ └── datax/
│ └── plugin/
│ └── writer/
│ ├── conn/
│ │ └── TSDBConnectionTest.java
│ └── util/
│ ├── Const.java
│ ├── HttpUtilsTest.java
│ └── TSDBTest.java
├── txtfilereader/
│ ├── doc/
│ │ └── txtfilereader.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── reader/
│ │ └── txtfilereader/
│ │ ├── Constant.java
│ │ ├── Key.java
│ │ ├── TxtFileReader.java
│ │ └── TxtFileReaderErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
├── txtfilewriter/
│ ├── doc/
│ │ └── txtfilewriter.md
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── assembly/
│ │ └── package.xml
│ ├── java/
│ │ └── com/
│ │ └── alibaba/
│ │ └── datax/
│ │ └── plugin/
│ │ └── writer/
│ │ └── txtfilewriter/
│ │ ├── Key.java
│ │ ├── TxtFileWriter.java
│ │ └── TxtFileWriterErrorCode.java
│ └── resources/
│ ├── plugin.json
│ └── plugin_job_template.json
└── userGuid.md
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# Created by .ignore support plugin (hsz.mobi)
.DS_Store
.AppleDouble
.LSOverride
Icon
._*
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
*.class
*.log
*.ctxt
.mtj.tmp/
*.jar
*.war
*.nar
*.ear
*.zip
*.tar.gz
*.rar
hs_err_pid*
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/dictionaries
.idea/**/shelf
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
.idea/**/gradle.xml
.idea/**/libraries
cmake-build-debug/
cmake-build-release/
.idea/**/mongoSettings.xml
*.iws
out/
.idea_modules/
atlassian-ide-plugin.xml
.idea/replstate.xml
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
.idea/httpRequests
target/
pom.xml.tag
pom.xml.releaseBackup
pom.xml.versionsBackup
pom.xml.next
release.properties
dependency-reduced-pom.xml
buildNumber.properties
.mvn/timing.properties
!/.mvn/wrapper/maven-wrapper.jar
.idea
*.iml
out
gen### Python template
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
*.manifest
*.spec
pip-log.txt
pip-delete-this-directory.txt
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
*.mo
*.pot
*.log
local_settings.py
db.sqlite3
instance/
.webassets-cache
.scrapy
docs/_build/
target/
.ipynb_checkpoints
.python-version
celerybeat-schedule
*.sage.py
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
.spyderproject
.spyproject
.ropeproject
/site
.mypy_cache/
.metadata
bin/
tmp/
*.tmp
*.bak
*.swp
*~.nib
local.properties
.settings/
.loadpath
.recommenders
.externalToolBuilders/
*.launch
*.pydevproject
.cproject
.autotools
.factorypath
.buildpath
.target
.tern-project
.texlipse
.springBeans
.recommenders/
.cache-main
.scala_dependencies
.worksheet
================================================
FILE: NOTICE
================================================
========================================================
DataX 是阿里云 DataWorks数据集成 的开源版本，在阿里巴巴集团内被广泛使用的离线数据同步工具/平台。DataX 实现了包括 MySQL、Oracle、OceanBase、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、Hologres、DRDS 等各种异构数据源之间高效的数据同步功能。
DataX is an open source offline data synchronization tool / platform widely used in Alibaba group and other companies. DataX implements efficient data synchronization between heterogeneous data sources including mysql, Oracle, oceanbase, sqlserver, postgre, HDFS, hive, ads, HBase, tablestore (OTS), maxcompute (ODPs), hologres, DRDS, etc.
Copyright 1999-2022 Alibaba Group Holding Ltd.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
===================================================================
文级别引用，按许可证
This product contains various third-party components under other open source licenses.
This section summarizes those components and their licenses.
GNU Lesser General Public License
--------------------------------------
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/CliQuery.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/Connection4TSDB.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/DataPoint4TSDB.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/DumpSeries.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/OpenTSDBConnection.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/OpenTSDBDump.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/Constant.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/Key.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/OpenTSDBReader.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/OpenTSDBReaderErrorCode.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/util/HttpUtils.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/util/TSDBUtils.java
opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/util/TimeUtils.java
===================================================================
================================================
FILE: README.md
================================================
# DataX
[](https://opensource.alibaba.com/contribution_leaderboard/details?projectValue=datax)
DataX 是阿里云 [DataWorks数据集成](https://www.aliyun.com/product/bigdata/ide) 的开源版本，在阿里巴巴集团内被广泛使用的离线数据同步工具/平台。DataX 实现了包括 MySQL、Oracle、OceanBase、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、Hologres、DRDS, databend 等各种异构数据源之间高效的数据同步功能。
# DataX 商业版本
阿里云DataWorks数据集成是DataX团队在阿里云上的商业化产品，致力于提供复杂网络环境下、丰富的异构数据源之间高速稳定的数据移动能力，以及繁杂业务背景下的数据同步解决方案。目前已经支持云上近3000家客户，单日同步数据超过3万亿条。DataWorks数据集成目前支持离线50+种数据源，可以进行整库迁移、批量上云、增量同步、分库分表等各类同步解决方案。2020年更新实时同步能力，支持10+种数据源的读写任意组合。提供MySQL，Oracle等多种数据源到阿里云MaxCompute，Hologres等大数据引擎的一键全增量同步解决方案。
商业版本参见： https://www.aliyun.com/product/bigdata/ide
# Features
DataX本身作为数据同步框架，将不同数据源的同步抽象为从源头数据源读取数据的Reader插件，以及向目标端写入数据的Writer插件，理论上DataX框架可以支持任意数据源类型的数据同步工作。同时DataX插件体系作为一套生态系统, 每接入一套新数据源该新加入的数据源即可实现和现有的数据源互通。
# DataX详细介绍
##### 请参考：[DataX-Introduction](https://github.com/alibaba/DataX/blob/master/introduction.md)
# Quick Start
##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202308/datax.tar.gz)
##### 请点击：[Quick Start](https://github.com/alibaba/DataX/blob/master/userGuid.md)
# Support Data Channels
DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、NOSQL、大数据计算系统都已经接入，目前支持数据如下图，详情请点击：[DataX数据源参考指南](https://github.com/alibaba/DataX/wiki/DataX-all-data-channels)
| 类型 | 数据源 | Reader(读) | Writer(写) | 文档 |
|--------------|---------------------------|:---------:|:---------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
| RDBMS 关系型数据库 | MySQL | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/mysqlreader/doc/mysqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/mysqlwriter/doc/mysqlwriter.md) |
| | Oracle | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/oraclereader/doc/oraclereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/oraclewriter/doc/oraclewriter.md) |
| | OceanBase | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/oceanbasev10reader/doc/oceanbasev10reader.md) 、[写](https://github.com/alibaba/DataX/blob/master/oceanbasev10writer/doc/oceanbasev10writer.md) |
| | SQLServer | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/sqlserverreader/doc/sqlserverreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/sqlserverwriter/doc/sqlserverwriter.md) |
| | PostgreSQL | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/postgresqlreader/doc/postgresqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/postgresqlwriter/doc/postgresqlwriter.md) |
| | DRDS | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/drdsreader/doc/drdsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/drdswriter/doc/drdswriter.md) |
| | Kingbase | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/drdsreader/doc/drdsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/drdswriter/doc/drdswriter.md) |
| | 通用RDBMS(支持所有关系型数据库) | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/rdbmsreader/doc/rdbmsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/rdbmswriter/doc/rdbmswriter.md) |
| 阿里云数仓数据存储 | ODPS | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/odpsreader/doc/odpsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/odpswriter/doc/odpswriter.md) |
| | ADB | | √ | [写](https://github.com/alibaba/DataX/blob/master/adbmysqlwriter/doc/adbmysqlwriter.md) |
| | ADS | | √ | [写](https://github.com/alibaba/DataX/blob/master/adswriter/doc/adswriter.md) |
| | OSS | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/ossreader/doc/ossreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/osswriter/doc/osswriter.md) |
| | OCS | | √ | [写](https://github.com/alibaba/DataX/blob/master/ocswriter/doc/ocswriter.md) |
| | Hologres | | √ | [写](https://github.com/alibaba/DataX/blob/master/hologresjdbcwriter/doc/hologresjdbcwriter.md) |
| | AnalyticDB For PostgreSQL | | √ | 写 |
| 阿里云中间件 | datahub | √ | √ | 读 、写 |
| | SLS | √ | √ | 读 、写 |
| 图数据库 | 阿里云 GDB | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/gdbreader/doc/gdbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/gdbwriter/doc/gdbwriter.md) |
| | Neo4j | | √ | [写](https://github.com/alibaba/DataX/blob/master/neo4jwriter/doc/neo4jwriter.md) |
| NoSQL数据存储 | OTS | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/otsreader/doc/otsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/otswriter/doc/otswriter.md) |
| | Hbase0.94 | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hbase094xreader/doc/hbase094xreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hbase094xwriter/doc/hbase094xwriter.md) |
| | Hbase1.1 | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hbase11xreader/doc/hbase11xreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hbase11xwriter/doc/hbase11xwriter.md) |
| | Phoenix4.x | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hbase11xsqlreader/doc/hbase11xsqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hbase11xsqlwriter/doc/hbase11xsqlwriter.md) |
| | Phoenix5.x | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hbase20xsqlreader/doc/hbase20xsqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hbase20xsqlwriter/doc/hbase20xsqlwriter.md) |
| | MongoDB | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/mongodbreader/doc/mongodbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/mongodbwriter/doc/mongodbwriter.md) |
| | Cassandra | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/cassandrareader/doc/cassandrareader.md) 、[写](https://github.com/alibaba/DataX/blob/master/cassandrawriter/doc/cassandrawriter.md) |
| 数仓数据存储 | StarRocks | √ | √ | 读 、[写](https://github.com/alibaba/DataX/blob/master/starrockswriter/doc/starrockswriter.md) |
| | ApacheDoris | | √ | [写](https://github.com/alibaba/DataX/blob/master/doriswriter/doc/doriswriter.md) |
| | ClickHouse | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/clickhousereader/doc/clickhousereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/clickhousewriter/doc/clickhousewriter.md) |
| | Databend | | √ | [写](https://github.com/alibaba/DataX/blob/master/databendwriter/doc/databendwriter.md) |
| | Hive | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md) |
| | kudu | | √ | [写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md) |
| | selectdb | | √ | [写](https://github.com/alibaba/DataX/blob/master/selectdbwriter/doc/selectdbwriter.md) |
| 无结构化数据存储 | TxtFile | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/txtfilereader/doc/txtfilereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/txtfilewriter/doc/txtfilewriter.md) |
| | FTP | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/ftpreader/doc/ftpreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/ftpwriter/doc/ftpwriter.md) |
| | HDFS | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md) |
| | Elasticsearch | | √ | [写](https://github.com/alibaba/DataX/blob/master/elasticsearchwriter/doc/elasticsearchwriter.md) |
| 时间序列数据库 | OpenTSDB | √ | | [读](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md) |
| | TSDB | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/tsdbreader/doc/tsdbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md) |
| | TDengine | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/tdenginereader/doc/tdenginereader-CN.md) 、[写](https://github.com/alibaba/DataX/blob/master/tdenginewriter/doc/tdenginewriter-CN.md) |
# 阿里云DataWorks数据集成
目前DataX的已有能力已经全部融和进阿里云的数据集成，并且比DataX更加高效、安全，同时数据集成具备DataX不具备的其它高级特性和功能。可以理解为数据集成是DataX的全面升级的商业化用版本，为企业可以提供稳定、可靠、安全的数据传输服务。与DataX相比，数据集成主要有以下几大突出特点：
支持实时同步：
- 功能简介：https://help.aliyun.com/document_detail/181912.html
- 支持的数据源：https://help.aliyun.com/document_detail/146778.html
- 支持数据处理：https://help.aliyun.com/document_detail/146777.html
离线同步数据源种类大幅度扩充：
- 新增比如：DB2、Kafka、Hologres、MetaQ、SAPHANA、达梦等等，持续扩充中
- 离线同步支持的数据源：https://help.aliyun.com/document_detail/137670.html
- 具备同步解决方案：
- 解决方案系统：https://help.aliyun.com/document_detail/171765.html
- 一键全增量：https://help.aliyun.com/document_detail/175676.html
- 整库迁移：https://help.aliyun.com/document_detail/137809.html
- 批量上云：https://help.aliyun.com/document_detail/146671.html
- 更新更多能力请访问：https://help.aliyun.com/document_detail/137663.html
-
# 我要开发新的插件
请点击：[DataX插件开发宝典](https://github.com/alibaba/DataX/blob/master/dataxPluginDev.md)
# 重要版本更新说明
DataX 后续计划月度迭代更新，也欢迎感兴趣的同学提交 Pull requests，月度更新内容如下。
- [datax_v202309]（https://github.com/alibaba/DataX/releases/tag/datax_v202309)
- 支持Phoenix 同步数据添加 where条件
- 支持华为 GuassDB读写插件
- 修复ClickReader 插件运行报错 Can't find bundle for base name
- 增加 DataX调试模块
- 修复 orc空文件报错问题
- 优化obwriter性能
- txtfilewriter 增加导出为insert语句功能支持
- HdfsReader/HdfsWriter 支持parquet读写能力
- [datax_v202308]（https://github.com/alibaba/DataX/releases/tag/datax_v202308)
- OTS 插件更新
- databend 插件更新
- Oceanbase驱动修复
- [datax_v202306]（https://github.com/alibaba/DataX/releases/tag/datax_v202306)
- 精简代码
- 新增插件（neo4jwriter、clickhousewriter）
- 优化插件、修复问题（oceanbase、hdfs、databend、txtfile）
- [datax_v202303]（https://github.com/alibaba/DataX/releases/tag/datax_v202303)
- 精简代码
- 新增插件（adbmysqlwriter、databendwriter、selectdbwriter）
- 优化插件、修复问题（sqlserver、hdfs、cassandra、kudu、oss）
- fastjson 升级到 fastjson2
- [datax_v202210]（https://github.com/alibaba/DataX/releases/tag/datax_v202210)
- 涉及通道能力更新（OceanBase、Tdengine、Doris等）
- [datax_v202209]（https://github.com/alibaba/DataX/releases/tag/datax_v202209)
- 涉及通道能力更新（MaxCompute、Datahub、SLS等）、安全漏洞更新、通用打包更新等
- [datax_v202205]（https://github.com/alibaba/DataX/releases/tag/datax_v202205)
- 涉及通道能力更新（MaxCompute、Hologres、OSS、Tdengine等）、安全漏洞更新、通用打包更新等
# 项目成员
核心Contributions: 言柏 、枕水、秋奇、青砾、一斅、云时
感谢天烬、光戈、祁然、巴真、静行对DataX做出的贡献。
# License
This software is free to use under the Apache License [Apache license](https://github.com/alibaba/DataX/blob/master/license.txt).
#
请及时提出issue给我们。请前往：[DataxIssue](https://github.com/alibaba/DataX/issues)
# 开源版DataX企业用户
```
长期招聘 联系邮箱：datax@alibabacloud.com
【JAVA开发职位】
职位名称：JAVA资深开发工程师/专家/高级专家
工作年限 : 2年以上
学历要求 : 本科（如果能力靠谱，这些都不是条件）
期望层级 : P6/P7/P8
岗位描述：
1. 负责阿里云大数据平台（数加）的开发设计。
2. 负责面向政企客户的大数据相关产品开发；
3. 利用大规模机器学习算法挖掘数据之间的联系，探索数据挖掘技术在实际场景中的产品应用 ；
4. 一站式大数据开发平台
5. 大数据任务调度引擎
6. 任务执行引擎
7. 任务监控告警
8. 海量异构数据同步
岗位要求：
1. 拥有3年以上JAVA Web开发经验；
2. 熟悉Java的基础技术体系。包括JVM、类装载、线程、并发、IO资源管理、网络；
3. 熟练使用常用Java技术框架、对新技术框架有敏锐感知能力；深刻理解面向对象、设计原则、封装抽象；
4. 熟悉HTML/HTML5和JavaScript；熟悉SQL语言；
5. 执行力强，具有优秀的团队合作精神、敬业精神；
6. 深刻理解设计模式及应用场景者加分；
7. 具有较强的问题分析和处理能力、比较强的动手能力，对技术有强烈追求者优先考虑；
8. 对高并发、高稳定可用性、高性能、大数据处理有过实际项目及产品经验者优先考虑；
9. 有大数据产品、云产品、中间件技术解决方案者优先考虑。
````
用户咨询支持：
钉钉群目前暂时受到了一些管控策略影响，建议大家有问题优先在这里提交问题 Issue，DataX研发和社区会定期回答Issue中的问题，知识库丰富后也能帮助到后来的使用者。
================================================
FILE: adbmysqlwriter/doc/adbmysqlwriter.md
================================================
# DataX AdbMysqlWriter
---
## 1 快速介绍
AdbMysqlWriter 插件实现了写入数据到 ADB MySQL 目的表的功能。在底层实现上， AdbMysqlWriter 通过 JDBC 连接远程 ADB MySQL 数据库，并执行相应的 `insert into ...` 或者 ( `replace into ...` ) 的 SQL 语句将数据写入 ADB MySQL，内部会分批次提交入库。
AdbMysqlWriter 面向ETL开发工程师，他们使用 AdbMysqlWriter 从数仓导入数据到 ADB MySQL。同时 AdbMysqlWriter 亦可以作为数据迁移工具为DBA等用户提供服务。
## 2 实现原理
AdbMysqlWriter 通过 DataX 框架获取 Reader 生成的协议数据，AdbMysqlWriter 通过 JDBC 连接远程 ADB MySQL 数据库，并执行相应的 `insert into ...` 或者 ( `replace into ...` ) 的 SQL 语句将数据写入 ADB MySQL。
* `insert into...`(遇到主键重复时会自动忽略当前写入数据，不做更新，作用等同于`insert ignore into`)
##### 或者
* `replace into...`(没有遇到主键/唯一性索引冲突时，与 insert into 行为一致，冲突时会用新行替换原有行所有字段) 的语句写入数据到 MySQL。出于性能考虑，采用了 `PreparedStatement + Batch`，并且设置了：`rewriteBatchedStatements=true`，将数据缓冲到线程上下文 Buffer 中，当 Buffer 累计到预定阈值时，才发起写入请求。
<br />
注意：整个任务至少需要具备 `insert/replace into...` 的权限，是否需要其他权限，取决于你任务配置中在 preSql 和 postSql 中指定的语句。
## 3 功能说明
### 3.1 配置样例
* 这里使用一份从内存产生到 ADB MySQL 导入的数据。
```json
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
"name": "streamreader",
"parameter": {
"column" : [
{
"value": "DataX",
"type": "string"
},
{
"value": 19880808,
"type": "long"
},
{
"value": "1988-08-08 08:08:08",
"type": "date"
},
{
"value": true,
"type": "bool"
},
{
"value": "test",
"type": "bytes"
}
],
"sliceRecordCount": 1000
}
},
"writer": {
"name": "adbmysqlwriter",
"parameter": {
"writeMode": "replace",
"username": "root",
"password": "root",
"column": [
"*"
],
"preSql": [
"truncate table @table"
],
"connection": [
{
"jdbcUrl": "jdbc:mysql://ip:port/database?useUnicode=true",
"table": [
"test"
]
}
]
}
}
}
]
}
}
```
### 3.2 参数说明
* **jdbcUrl**
* 描述：目的数据库的 JDBC 连接信息。作业运行时，DataX 会在你提供的 jdbcUrl 后面追加如下属性：yearIsDateType=false&zeroDateTimeBehavior=convertToNull&rewriteBatchedStatements=true
注意：1、在一个数据库上只能配置一个 jdbcUrl
2、一个 AdbMySQL 写入任务仅能配置一个 jdbcUrl
3、jdbcUrl按照MySQL官方规范，并可以填写连接附加控制信息，比如想指定连接编码为 gbk ，则在 jdbcUrl 后面追加属性 useUnicode=true&characterEncoding=gbk。具体请参看 Mysql官方文档或者咨询对应 DBA。
* 必选：是 <br />
* 默认值：无 <br />
* **username**
* 描述：目的数据库的用户名 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **password**
* 描述：目的数据库的密码 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **table**
* 描述：目的表的表名称。只能配置一个 AdbMySQL 的表名称。
注意：table 和 jdbcUrl 必须包含在 connection 配置单元中
* 必选：是 <br />
* 默认值：无 <br />
* **column**
* 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id", "name", "age"]。如果要依次写入全部列，使用`*`表示, 例如: `"column": ["*"]`。
**column配置项必须指定，不能留空！**
注意：1、我们强烈不推荐你这样配置，因为当你目的表字段个数、类型等有改动时，你的任务可能运行不正确或者失败
2、 column 不能配置任何常量值
* 必选：是 <br />
* 默认值：否 <br />
* **session**
* 描述: DataX在获取 ADB MySQL 连接时，执行session指定的SQL语句，修改当前connection session属性
* 必须: 否
* 默认值: 空
* **preSql**
* 描述：写入数据到目的表前，会先执行这里的标准语句。如果 Sql 中有你需要操作到的表名称，请使用 `@table` 表示，这样在实际执行 SQL 语句时，会对变量按照实际表名称进行替换。比如希望导入数据前，先对表中数据进行删除操作，那么你可以这样配置：`"preSql":["truncate table @table"]`，效果是：在执行到每个表写入数据前，会先执行对应的 `truncate table 对应表名称` <br />
* 必选：否 <br />
* 默认值：无 <br />
* **postSql**
* 描述：写入数据到目的表后，会执行这里的标准语句。（原理同 preSql ） <br />
* 必选：否 <br />
* 默认值：无 <br />
* **writeMode**
* 描述：控制写入数据到目标表采用 `insert into` 或者 `replace into` 或者 `ON DUPLICATE KEY UPDATE` 语句<br />
* 必选：是 <br />
* 所有选项：insert/replace/update <br />
* 默认值：replace <br />
* **batchSize**
* 描述：一次性批量提交的记录数大小，该值可以极大减少DataX与 Adb MySQL 的网络交互次数，并提升整体吞吐量。但是该值设置过大可能会造成DataX运行进程OOM情况。<br />
* 必选：否 <br />
* 默认值：2048 <br />
### 3.3 类型转换
目前 AdbMysqlWriter 支持大部分 MySQL 类型，但也存在部分个别类型没有支持的情况，请注意检查你的类型。
下面列出 AdbMysqlWriter 针对 MySQL 类型转换列表:
| DataX 内部类型 | AdbMysql 数据类型 |
|---------------|---------------------------------|
| Long | tinyint, smallint, int, bigint |
| Double | float, double, decimal |
| String | varchar |
| Date | date, time, datetime, timestamp |
| Boolean | boolean |
| Bytes | binary |
## 4 性能报告
### 4.1 环境准备
#### 4.1.1 数据特征
TPC-H 数据集 lineitem 表，共 17 个字段, 随机生成总记录行数 59986052。未压缩总数据量：7.3GiB
建表语句：
CREATE TABLE `datax_adbmysqlwriter_perf_lineitem` (
`l_orderkey` bigint NOT NULL COMMENT '',
`l_partkey` int NOT NULL COMMENT '',
`l_suppkey` int NOT NULL COMMENT '',
`l_linenumber` int NOT NULL COMMENT '',
`l_quantity` decimal(15,2) NOT NULL COMMENT '',
`l_extendedprice` decimal(15,2) NOT NULL COMMENT '',
`l_discount` decimal(15,2) NOT NULL COMMENT '',
`l_tax` decimal(15,2) NOT NULL COMMENT '',
`l_returnflag` varchar(1024) NOT NULL COMMENT '',
`l_linestatus` varchar(1024) NOT NULL COMMENT '',
`l_shipdate` date NOT NULL COMMENT '',
`l_commitdate` date NOT NULL COMMENT '',
`l_receiptdate` date NOT NULL COMMENT '',
`l_shipinstruct` varchar(1024) NOT NULL COMMENT '',
`l_shipmode` varchar(1024) NOT NULL COMMENT '',
`l_comment` varchar(1024) NOT NULL COMMENT '',
`dummy` varchar(1024),
PRIMARY KEY (`l_orderkey`, `l_linenumber`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='datax perf test';
单行记录类似于：
l_orderkey: 2122789
l_partkey: 1233571
l_suppkey: 8608
l_linenumber: 1
l_quantity: 35.00
l_extendedprice: 52657.85
l_discount: 0.02
l_tax: 0.07
l_returnflag: N
l_linestatus: O
l_shipdate: 1996-11-03
l_commitdate: 1996-12-07
l_receiptdate: 1996-11-16
l_shipinstruct: COLLECT COD
l_shipmode: FOB
l_comment: ld, regular theodolites.
dummy:
#### 4.1.2 机器参数
* DataX ECS: 24Core48GB
* Adb MySQL 数据库
* 计算资源：16Core64GB（集群版）
* 弹性IO资源：3
#### 4.1.3 DataX jvm 参数
-Xms1G -Xmx10G -XX:+HeapDumpOnOutOfMemoryError
### 4.2 测试报告
| 通道数 | 批量提交行数 | DataX速度(Rec/s) | DataX流量(MB/s) | 导入用时(s) |
|-----|-------|------------------|---------------|---------|
| 1 | 512 | 23071 | 2.34 | 2627 |
| 1 | 1024 | 26080 | 2.65 | 2346 |
| 1 | 2048 | 28162 | 2.86 | 2153 |
| 1 | 4096 | 28978 | 2.94 | 2119 |
| 4 | 512 | 56590 | 5.74 | 1105 |
| 4 | 1024 | 81062 | 8.22 | 763 |
| 4 | 2048 | 107117 | 10.87 | 605 |
| 4 | 4096 | 113181 | 11.48 | 579 |
| 8 | 512 | 81062 | 8.22 | 786 |
| 8 | 1024 | 127629 | 12.95 | 519 |
| 8 | 2048 | 187456 | 19.01 | 369 |
| 8 | 4096 | 206848 | 20.98 | 341 |
| 16 | 512 | 130404 | 13.23 | 513 |
| 16 | 1024 | 214235 | 21.73 | 335 |
| 16 | 2048 | 299930 | 30.42 | 253 |
| 16 | 4096 | 333255 | 33.80 | 227 |
| 32 | 512 | 206848 | 20.98 | 347 |
| 32 | 1024 | 315716 | 32.02 | 241 |
| 32 | 2048 | 399907 | 40.56 | 199 |
| 32 | 4096 | 461431 | 46.80 | 184 |
| 64 | 512 | 333255 | 33.80 | 231 |
| 64 | 1024 | 399907 | 40.56 | 204 |
| 64 | 2048 | 428471 | 43.46 | 199 |
| 64 | 4096 | 461431 | 46.80 | 187 |
| 128 | 512 | 333255 | 33.80 | 235 |
| 128 | 1024 | 399907 | 40.56 | 203 |
| 128 | 2048 | 425432 | 43.15 | 197 |
| 128 | 4096 | 387006 | 39.26 | 211 |
说明：
1. datax 使用 txtfilereader 读取本地文件，避免源端存在性能瓶颈。
#### 性能测试小结
1. channel通道个数和batchSize对性能影响比较大
2. 通常不建议写入数据库时，通道个数 > 32
## 5 约束限制
## FAQ
***
**Q: AdbMysqlWriter 执行 postSql 语句报错，那么数据导入到目标数据库了吗?**
A: DataX 导入过程存在三块逻辑，pre 操作、导入操作、post 操作，其中任意一环报错，DataX 作业报错。由于 DataX 不能保证在同一个事务完成上述几个操作，因此有可能数据已经落入到目标端。
***
**Q: 按照上述说法，那么有部分脏数据导入数据库，如果影响到线上数据库怎么办?**
A: 目前有两种解法，第一种配置 pre 语句，该 sql 可以清理当天导入数据， DataX 每次导入时候可以把上次清理干净并导入完整数据。第二种，向临时表导入数据，完成后再 rename 到线上表。
***
**Q: 上面第二种方法可以避免对线上数据造成影响，那我具体怎样操作?**
A: 可以配置临时表导入
================================================
FILE: adbmysqlwriter/pom.xml
================================================
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-all</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<artifactId>adbmysqlwriter</artifactId>
<name>adbmysqlwriter</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-common</artifactId>
<version>${datax-project-version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>plugin-rdbms-util</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.40</version>
</dependency>
</dependencies>
<build>
<plugins>
<!-- compiler plugin -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${jdk-version}</source>
<target>${jdk-version}</target>
<encoding>${project-sourceEncoding}</encoding>
</configuration>
</plugin>
<!-- assembly plugin -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assembly/package.xml</descriptor>
</descriptors>
<finalName>datax</finalName>
</configuration>
<executions>
<execution>
<id>dwzip</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: adbmysqlwriter/src/main/assembly/package.xml
================================================
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id></id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>plugin.json</include>
<include>plugin_job_template.json</include>
</includes>
<outputDirectory>plugin/writer/adbmysqlwriter</outputDirectory>
</fileSet>
<fileSet>
<directory>target/</directory>
<includes>
<include>adbmysqlwriter-0.0.1-SNAPSHOT.jar</include>
</includes>
<outputDirectory>plugin/writer/adbmysqlwriter</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>plugin/writer/adbmysqlwriter/libs</outputDirectory>
<scope>runtime</scope>
</dependencySet>
</dependencySets>
</assembly>
================================================
FILE: adbmysqlwriter/src/main/java/com/alibaba/datax/plugin/writer/adbmysqlwriter/AdbMysqlWriter.java
================================================
package com.alibaba.datax.plugin.writer.adbmysqlwriter;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.spi.Writer;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter;
import com.alibaba.datax.plugin.rdbms.writer.Key;
import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
public class AdbMysqlWriter extends Writer {
private static final DataBaseType DATABASE_TYPE = DataBaseType.ADB;
public static class Job extends Writer.Job {
private Configuration originalConfig = null;
private CommonRdbmsWriter.Job commonRdbmsWriterJob;
@Override
public void preCheck(){
this.init();
this.commonRdbmsWriterJob.writerPreCheck(this.originalConfig, DATABASE_TYPE);
}
@Override
public void init() {
this.originalConfig = super.getPluginJobConf();
this.commonRdbmsWriterJob = new CommonRdbmsWriter.Job(DATABASE_TYPE);
this.commonRdbmsWriterJob.init(this.originalConfig);
}
// 一般来说，是需要推迟到 task 中进行pre 的执行（单表情况例外）
@Override
public void prepare() {
//实跑先不支持 权限 检验
//this.commonRdbmsWriterJob.privilegeValid(this.originalConfig, DATABASE_TYPE);
this.commonRdbmsWriterJob.prepare(this.originalConfig);
}
@Override
public List<Configuration> split(int mandatoryNumber) {
return this.commonRdbmsWriterJob.split(this.originalConfig, mandatoryNumber);
}
// 一般来说，是需要推迟到 task 中进行post 的执行（单表情况例外）
@Override
public void post() {
this.commonRdbmsWriterJob.post(this.originalConfig);
}
@Override
public void destroy() {
this.commonRdbmsWriterJob.destroy(this.originalConfig);
}
}
public static class Task extends Writer.Task {
private Configuration writerSliceConfig;
private CommonRdbmsWriter.Task commonRdbmsWriterTask;
public static class DelegateClass extends CommonRdbmsWriter.Task {
private long writeTime = 0L;
private long writeCount = 0L;
private long lastLogTime = 0;
public DelegateClass(DataBaseType dataBaseType) {
super(dataBaseType);
}
@Override
protected void doBatchInsert(Connection connection, List<Record> buffer)
throws SQLException {
long startTime = System.currentTimeMillis();
super.doBatchInsert(connection, buffer);
writeCount = writeCount + buffer.size();
writeTime = writeTime + (System.currentTimeMillis() - startTime);
// log write metrics every 10 seconds
if (System.currentTimeMillis() - lastLogTime > 10000) {
lastLogTime = System.currentTimeMillis();
logTotalMetrics();
}
}
public void logTotalMetrics() {
LOG.info(Thread.currentThread().getName() + ", AdbMySQL writer take " + writeTime + " ms, write " + writeCount + " records.");
}
}
@Override
public void init() {
this.writerSliceConfig = super.getPluginJobConf();
if (StringUtils.isBlank(this.writerSliceConfig.getString(Key.WRITE_MODE))) {
this.writerSliceConfig.set(Key.WRITE_MODE, "REPLACE");
}
this.commonRdbmsWriterTask = new DelegateClass(DATABASE_TYPE);
this.commonRdbmsWriterTask.init(this.writerSliceConfig);
}
@Override
public void prepare() {
this.commonRdbmsWriterTask.prepare(this.writerSliceConfig);
}
//TODO 改用连接池，确保每次获取的连接都是可用的（注意：连接可能需要每次都初始化其 session）
public void startWrite(RecordReceiver recordReceiver) {
this.commonRdbmsWriterTask.startWrite(recordReceiver, this.writerSliceConfig,
super.getTaskPluginCollector());
}
@Override
public void post() {
this.commonRdbmsWriterTask.post(this.writerSliceConfig);
}
@Override
public void destroy() {
this.commonRdbmsWriterTask.destroy(this.writerSliceConfig);
}
@Override
public boolean supportFailOver(){
String writeMode = writerSliceConfig.getString(Key.WRITE_MODE);
return "replace".equalsIgnoreCase(writeMode);
}
}
}
================================================
FILE: adbmysqlwriter/src/main/resources/plugin.json
================================================
{
"name": "adbmysqlwriter",
"class": "com.alibaba.datax.plugin.writer.adbmysqlwriter.AdbMysqlWriter",
"description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.",
"developer": "alibaba"
}
================================================
FILE: adbmysqlwriter/src/main/resources/plugin_job_template.json
================================================
{
"name": "adbmysqlwriter",
"parameter": {
"username": "username",
"password": "password",
"column": ["col1", "col2", "col3"],
"connection": [
{
"jdbcUrl": "jdbc:mysql://<host>:<port>[/<database>]",
"table": ["table1", "table2"]
}
],
"preSql": [],
"postSql": [],
"batchSize": 65536,
"batchByteSize": 134217728,
"dryRun": false,
"writeMode": "insert"
}
}
================================================
FILE: adbpgwriter/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>datax-all</artifactId>
<groupId>com.alibaba.datax</groupId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>adbpgwriter</artifactId>
<name>adbpgwriter</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-common</artifactId>
<version>${datax-project-version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-core</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>plugin-rdbms-util</artifactId>
<version>${datax-project-version}</version>
<exclusions>
<exclusion>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.1.17</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>com.alibaba.cloud.analyticdb</groupId>
<artifactId>adb4pgclient</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<!-- compiler plugin -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${jdk-version}</source>
<target>${jdk-version}</target>
<encoding>${project-sourceEncoding}</encoding>
</configuration>
</plugin>
<!-- assembly plugin -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assembly/package.xml</descriptor>
</descriptors>
<finalName>datax</finalName>
</configuration>
<executions>
<execution>
<id>dwzip</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: adbpgwriter/src/main/assembly/package.xml
================================================
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id></id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>plugin.json</include>
<include>plugin_job_template.json</include>
</includes>
<outputDirectory>plugin/writer/adbpgwriter</outputDirectory>
</fileSet>
<fileSet>
<directory>target/</directory>
<includes>
<include>adbpgwriter-0.0.1-SNAPSHOT.jar</include>
</includes>
<outputDirectory>plugin/writer/adbpgwriter</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>plugin/writer/adbpgwriter/libs</outputDirectory>
<scope>runtime</scope>
</dependencySet>
</dependencySets>
</assembly>
================================================
FILE: adbpgwriter/src/main/doc/adbpgwriter.md
================================================
# DataX ADB PG Writer
---
## 1 快速介绍
AdbpgWriter 插件实现了写入数据到 ABD PG版数据库的功能。在底层实现上，AdbpgWriter 插件会先缓存需要写入的数据，当缓存的
数据量达到 commitSize 时，插件会通过 JDBC 连接远程 ADB PG版 数据库，并执行 COPY 命令将数据写入 ADB PG 数据库。
AdbpgWriter 可以作为数据迁移工具为用户提供服务。
## 2 实现原理
AdbpgWriter 通过 DataX 框架获取 Reader 生成的协议数据，首先会将数据缓存，当缓存的数据量达到commitSize时，插件根据你配置生成相应的COPY语句，执行
COPY命令将数据写入ADB PG数据库中。
## 3 功能说明
### 3.1 配置样例
* 这里使用一份从内存产生到 AdbpgWriter导入的数据
```json
{
"job": {
"setting": {
"speed": {
"channel": 32
}
},
"content": [
{
"reader": {
"name": "streamreader",
"parameter": {
"column" : [
{
"value": "DataX",
"type": "string"
},
{
"value": 19880808,
"type": "long"
},
{
"value": "1988-08-08 08:08:08",
"type": "date"
},
{
"value": true,
"type": "bool"
},
{
"value": "test",
"type": "bytes"
}
]
},
"sliceRecordCount": 1000
},
"writer": {
"name": "adbpgwriter",
"parameter": {
"username": "",
"password": "",
"host": "127.0.0.1",
"port": "1234",
"database": "database",
"schema": "schema",
"table": "table",
"preSql": ["delete * from table"],
"postSql": ["select * from table"],
"column": ["*"]
}
}
}
]
}
}
```
### 3.2 参数说明
* **name**
* 描述：插件名称 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **username**
* 描述：目的数据库的用户名 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **password**
* 描述：目的数据库的密码 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **host**
* 描述：目的数据库主机名 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **port**
* 描述：目的数据库的端口 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **database**
* 描述：需要写入的表所属的数据库名称 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **schema**
* 描述：需要写入的表所属的schema名称 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **table**
* 描述：需要写入的表名称 <br />
* 必选：是 <br />
* 默认值：无 <br />
* **column**
* 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。如果要依次写入全部列，使用*表示, 例如: "column": ["*"]
注意：1、我们强烈不推荐你这样配置，因为当你目的表字段个数、类型等有改动时，你的任务可能运行不正确或者失败
2、此处 column 不能配置任何常量值
3、大写字段名,此处配置时,不需要拼接转义符号:\"
* 必选：是 <br />
* 默认值：否 <br />
* **preSql**
* 描述：写入数据到目的表前，会先执行这里的标准语句。如果 Sql 中有你需要操作到的表名称，可以使用 `@table` 表示，这样在实际执行 Sql 语句时，会对变量按照实际表名称进行替换。比如你的任务是要写入到目的端的100个同构分表(表名称为:datax_00,datax01, ... datax_98,datax_99)，并且你希望导入数据前，先对表中数据进行删除操作，那么你可以这样配置：`"preSql":["delete from @table"]`，效果是：在执行到每个表写入数据前，会先执行对应的 delete from 对应表名称 <br />
* 必选：否 <br />
* 默认值：否 <br />
* **postSql**
* 描述：写入数据到目的表后，会先执行这里的标准语句。如果 Sql 中有你需要操作到的表名称，可以使用 `@table` 表示，这样在实际执行 Sql 语句时，会对变量按照实际表名称进行替换。 <br />
* 必选：否 <br />
* 默认值：否 <br />
### 3.3 类型转换
目前 AdbpgWriter 支持大部分 ADB PG 数据库的类型，但也存在部分没有支持的情况，请注意检查你的类型。
下面列出 AdbpgWriter 针对 ADB PG 类型转换列表:
| DataX 内部类型| ADB PG 数据类型 |
| -------- | ----- |
| Long |bigint, bigserial, integer, smallint, serial |
| Double |double precision, float, numeric, real |
| String |varchar, char, text|
| Date |date, time, timestamp |
| Boolean |bool|
## 4 性能报告
### 4.1 环境准备
#### 4.1.1 数据特征
建表语句：
```sql
create table schematest.test_datax (
t1 int,
t2 bigint,
t3 bigserial,
t4 float,
t5 timestamp,
t6 varchar
)distributed by(t1);
```
#### 4.1.2 机器参数
* 执行DataX的机器参数为:
1. cpu: 24核
2. mem: 96GB
* ADB PG数据库机器参数为:
1. 平均core数量:4
2. primary segment 数量: 4
3. 计算组数量:2
### 4.2 测试报告
#### 4.2.1 单表测试报告
| 通道数| commitSize MB | DataX速度(Rec/s)| DataX流量(M/s)
|--------|--------| --------|--------|
|1| 10 | 54098 | 15.54 |
|1| 20 | 55000 | 15.80 |
|4| 10 | 183333 | 52.66 |
|4| 20 | 173684 | 49.89 |
|8| 10 | 330000 | 94.79 |
|8| 20 | 300000 | 86.17 |
|16| 10 | 412500 | 118.48 |
|16| 20 | 366666 | 105.32 |
|32| 10 | 366666 | 105.32 |
#### 4.2.2 性能测试小结
1. `channel数对性能影响很大`
2. `通常不建议写入数据库时，通道个数 > 32`
================================================
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/AdbpgWriter.java
================================================
package com.alibaba.datax.plugin.writer.adbpgwriter;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.RecordSender;
import com.alibaba.datax.common.spi.Writer;
import com.alibaba.datax.common.util.Configuration;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter;
import com.alibaba.datax.plugin.rdbms.writer.Key;
import com.alibaba.datax.plugin.rdbms.writer.util.OriginalConfPretreatmentUtil;
import com.alibaba.datax.plugin.writer.adbpgwriter.copy.Adb4pgClientProxy;
import com.alibaba.datax.plugin.writer.adbpgwriter.util.Adb4pgUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode.*;
import static com.alibaba.datax.plugin.rdbms.util.DataBaseType.PostgreSQL;
/**
* @author yuncheng
*/
public class AdbpgWriter extends Writer {
private static final DataBaseType DATABASE_TYPE = DataBaseType.PostgreSQL;
public static class Job extends Writer.Job {
private Configuration originalConfig;
private CommonRdbmsWriter.Job commonRdbmsWriterMaster;
private static final Logger LOG = LoggerFactory.getLogger(Writer.Job.class);
@Override
public void init() {
this.originalConfig = super.getPluginJobConf();
LOG.info("in Job.init(), config is:[\n{}\n]", originalConfig.toJSON());
this.commonRdbmsWriterMaster = new CommonRdbmsWriter.Job(DATABASE_TYPE);
//convert to DatabaseConfig, use DatabaseConfig to check user configuration
Adb4pgUtil.checkConfig(originalConfig);
}
@Override
public void prepare() {
Adb4pgUtil.prepare(originalConfig);
}
@Override
public List<Configuration> split(int adviceNumber) {
List<Configuration> splitResult = new ArrayList<Configuration>();
for(int i = 0; i < adviceNumber; i++) {
splitResult.add(this.originalConfig.clone());
}
return splitResult;
}
@Override
public void post() {
Adb4pgUtil.post(originalConfig);
}
@Override
public void destroy() {
}
}
public static class Task extends Writer.Task {
private Configuration writerSliceConfig;
private CommonRdbmsWriter.Task commonRdbmsWriterSlave;
private Adb4pgClientProxy adb4pgClientProxy;
//Adb4pgClient client;
@Override
public void init() {
this.writerSliceConfig = super.getPluginJobConf();
this.adb4pgClientProxy = new Adb4pgClientProxy(writerSliceConfig, super.getTaskPluginCollector());
this.commonRdbmsWriterSlave = new CommonRdbmsWriter.Task(DATABASE_TYPE){
@Override
public String calcValueHolder(String columnType){
if("serial".equalsIgnoreCase(columnType)){
return "?::int";
}else if("bit".equalsIgnoreCase(columnType)){
return "?::bit varying";
}
return "?::" + columnType;
}
};
}
@Override
public void prepare() {
}
@Override
public void startWrite(RecordReceiver recordReceiver) {
this.adb4pgClientProxy.startWriteWithConnection(recordReceiver, Adb4pgUtil.getAdbpgConnect(writerSliceConfig));
}
@Override
public void post() {
}
@Override
public void destroy() {
}
}
}
================================================
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/copy/Adb4pgClientProxy.java
================================================
package com.alibaba.datax.plugin.writer.adbpgwriter.copy;
import com.alibaba.cloud.analyticdb.adb4pgclient.*;
import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.element.StringColumn;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.TaskPluginCollector;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.core.transport.record.DefaultRecord;
import com.alibaba.datax.plugin.rdbms.util.DBUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
import com.alibaba.datax.plugin.writer.adbpgwriter.util.Adb4pgUtil;
import com.alibaba.datax.plugin.writer.adbpgwriter.util.Constant;
import com.alibaba.datax.plugin.writer.adbpgwriter.util.Key;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
/**
* @author yuncheng
*/
public class Adb4pgClientProxy implements AdbProxy {
private static final Logger LOG = LoggerFactory.getLogger(Adb4pgClientProxy.class);
private Adb4pgClient adb4pgClient;
private String table;
private String schema;
List<String> columns;
private TableInfo tableInfo;
private TaskPluginCollector taskPluginCollector;
private boolean useRawData[];
public Adb4pgClientProxy(Configuration configuration,TaskPluginCollector taskPluginCollector) {
this.taskPluginCollector = taskPluginCollector;
DatabaseConfig databaseConfig = Adb4pgUtil.convertConfiguration(configuration);
// If the value of column is empty, set null
boolean emptyAsNull = configuration.getBool(Key.EMPTY_AS_NULL, false);
databaseConfig.setEmptyAsNull(emptyAsNull);
// 使用insert ignore into方式进行插入
boolean ignoreInsert = configuration.getBool(Key.IGNORE_INSERT, false);
databaseConfig.setInsertIgnore(ignoreInsert);
// commit时，写入ADB出现异常时重试的3次
int retryTimes = configuration.getInt(Key.RETRY_CONNECTION_TIME, Constant.DEFAULT_RETRY_TIMES);
databaseConfig.setRetryTimes(retryTimes);
// 重试间隔的时间为1s，单位是ms
int retryIntervalTime = configuration.getInt(Key.RETRY_INTERVAL_TIME, 1000);
databaseConfig.setRetryIntervalTime(retryIntervalTime);
// 设置自动提交的SQL长度（单位Byte），默认为10MB，一般不建议设置
int commitSize = configuration.getInt("commitSize", 10 * 1024 * 1024);
databaseConfig.setCommitSize(commitSize);
// 设置写入adb时的并发线程数，默认4，针对配置的所有表
int parallelNumber = configuration.getInt("parallelNumber", 4);
databaseConfig.setParallelNumber(parallelNumber);
// 设置client中使用的logger对象，此处使用slf4j.Logger
databaseConfig.setLogger(Adb4pgClientProxy.LOG);
// sdk 默认值为true
boolean shareDataSource = configuration.getBool("shareDataSource", true);
databaseConfig.setShareDataSource(shareDataSource);
//List<String> columns = configuration.getList(Key.COLUMN, String.class);
this.table = configuration.getString(com.alibaba.datax.plugin.rdbms.writer.Key.TABLE);
this.schema = configuration.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.SCHEMA);
this.adb4pgClient = new Adb4pgClient(databaseConfig);
this.columns = databaseConfig.getColumns(table,schema);
this.tableInfo = adb4pgClient.getTableInfo(table, schema);
this.useRawData = new boolean[this.columns.size()];
List<ColumnInfo> columnInfos = tableInfo.getColumns();
for (int i = 0; i < this.columns.size(); i++) {
String oriEachColumn = columns.get(i);
String eachColumn = oriEachColumn;
// 防御性保留字
if (eachColumn.startsWith(Constant.COLUMN_QUOTE_CHARACTER)
&& eachColumn.endsWith(Constant.COLUMN_QUOTE_CHARACTER)) {
eachColumn = eachColumn.substring(1, eachColumn.length() - 1);
}
for (ColumnInfo eachAdsColumn : columnInfos) {
if (eachColumn.equals(eachAdsColumn.getName())) {
int columnSqltype = eachAdsColumn.getDataType().sqlType;
switch (columnSqltype) {
case Types.DATE:
case Types.TIME:
case Types.TIMESTAMP:
this.useRawData[i] = false;
break;
default:
this.useRawData[i] = true;
break;
}
}
}
}
}
@Override
public void startWriteWithConnection(RecordReceiver recordReceiver, Connection connection) {
try {
Record record;
while ((record = recordReceiver.getFromReader()) != null) {
Row row = new Row();
List<Object> values = new ArrayList<Object>();
this.prepareColumnTypeValue(record, values);
row.setColumnValues(values);
try {
this.adb4pgClient.addRow(row,this.table, this.schema);
} catch (Adb4pgClientException e) {
if (101 == e.getCode()) {
for (String each : e.getErrData()) {
Record dirtyData = new DefaultRecord();
dirtyData.addColumn(new StringColumn(each));
this.taskPluginCollector.collectDirtyRecord(dirtyData, e.getMessage());
}
} else {
throw e;
}
}
}
try {
this.adb4pgClient.commit();
} catch (Adb4pgClientException e) {
if (101 == e.getCode()) {
for (String each : e.getErrData()) {
Record dirtyData = new DefaultRecord();
dirtyData.addColumn(new StringColumn(each));
this.taskPluginCollector.collectDirtyRecord(dirtyData, e.getMessage());
}
} else {
throw e;
}
}
}catch (Exception e) {
throw DataXException.asDataXException(DBUtilErrorCode.WRITE_DATA_ERROR, e);
}finally {
DBUtil.closeDBResources(null, null, connection);
}
return;
}
private void prepareColumnTypeValue(Record record, List<Object> values) {
for (int i = 0; i < this.columns.size(); i++) {
Column column = record.getColumn(i);
if (this.useRawData[i]) {
values.add(column.getRawData());
} else {
values.add(column.asString());
}
}
}
@Override
public void closeResource() {
try {
LOG.info("stop the adb4pgClient");
this.adb4pgClient.stop();
} catch (Exception e) {
LOG.warn("stop adbClient meet a exception, ignore it: {}", e.getMessage(), e);
}
}
}
================================================
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/copy/AdbProxy.java
================================================
package com.alibaba.datax.plugin.writer.adbpgwriter.copy;
import com.alibaba.datax.common.plugin.RecordReceiver;
import java.sql.Connection;
/**
* @author yuncheng
*/
public interface AdbProxy {
public abstract void startWriteWithConnection(RecordReceiver recordReceiver, Connection connection);
public void closeResource();
}
================================================
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/package-info.java
================================================
/**
* Greenplum Writer.
*
* @since 0.0.1
*/
package com.alibaba.datax.plugin.writer.adbpgwriter;
================================================
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Adb4pgUtil.java
================================================
package com.alibaba.datax.plugin.writer.adbpgwriter.util;
import com.alibaba.cloud.analyticdb.adb4pgclient.Adb4pgClient;
import com.alibaba.cloud.analyticdb.adb4pgclient.Adb4pgClientException;
import com.alibaba.cloud.analyticdb.adb4pgclient.DatabaseConfig;
import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.spi.ErrorCode;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.rdbms.util.DBUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
import com.alibaba.datax.plugin.rdbms.writer.Constant;
import com.alibaba.datax.plugin.rdbms.writer.Key;
import com.alibaba.datax.plugin.rdbms.writer.util.WriterUtil;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.util.*;
import static com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode.COLUMN_SPLIT_ERROR;
/**
* @author yuncheng
*/
public class Adb4pgUtil {
private static final Logger LOG = LoggerFactory.getLogger(Adb4pgUtil.class);
private static final DataBaseType DATABASE_TYPE = DataBaseType.PostgreSQL;
public static void checkConfig(Configuration originalConfig) {
try {
DatabaseConfig databaseConfig = convertConfiguration(originalConfig);
Adb4pgClient testConfigClient = new Adb4pgClient(databaseConfig);
} catch (Exception e) {
throw new Adb4pgClientException(Adb4pgClientException.CONFIG_ERROR, "Check config exception: " + e.getMessage(), null);
}
}
public static DatabaseConfig convertConfiguration(Configuration originalConfig) {
originalConfig.getNecessaryValue(Key.USERNAME, COLUMN_SPLIT_ERROR);
originalConfig.getNecessaryValue(Key.PASSWORD, COLUMN_SPLIT_ERROR);
String userName = originalConfig.getString(Key.USERNAME);
String passWord = originalConfig.getString(Key.PASSWORD);
String tableName = originalConfig.getString(Key.TABLE);
String schemaName = originalConfig.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.SCHEMA);
String host = originalConfig.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.HOST);
String port = originalConfig.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.PORT);
String databseName = originalConfig.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.DATABASE);
List<String> columns = originalConfig.getList(Key.COLUMN, String.class);
DatabaseConfig databaseConfig = new DatabaseConfig();
databaseConfig.setHost(host);
databaseConfig.setPort(Integer.valueOf(port));
databaseConfig.setDatabase(databseName);
databaseConfig.setUser(userName);
databaseConfig.setPassword(passWord);
databaseConfig.setLogger(LOG);
databaseConfig.setInsertIgnore(originalConfig.getBool(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.IS_INSERTINGORE, true));
databaseConfig.addTable(Collections.singletonList(tableName), schemaName);
databaseConfig.setColumns(columns, tableName, schemaName);
return databaseConfig;
}
private static Map<String, List<String>> splitBySchemaName(List<String> tables) {
HashMap<String, List<String>> res = new HashMap<String, List<String>>(16);
for (String schemaNameTableName: tables) {
String[] s = schemaNameTableName.split("\\.");
if (!res.containsKey(s[0])) {
res.put(s[0], new ArrayList<String>());
}
res.get(s[0]).add(s[1]);
}
return res;
}
public static Connection getAdbpgConnect(Configuration conf) {
String userName = conf.getString(Key.USERNAME);
String passWord = conf.getString(Key.PASSWORD);
return DBUtil.getConnection(DataBaseType.PostgreSQL, generateJdbcUrl(conf), userName, passWord);
}
private static String generateJdbcUrl(Configuration configuration) {
String host = configuration.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.HOST);
String port = configuration.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.PORT);
String databseName = configuration.getString(com.alibaba.datax.plugin.writer.adbpgwriter.util.Key.DATABASE);
String jdbcUrl = "jdbc:postgresql://" + host + ":" + port + "/" + databseName;
return jdbcUrl;
}
public static void prepare(Configuration originalConfig) {
List<String> preSqls = originalConfig.getList(Key.PRE_SQL,
String.class);
String tableName = originalConfig.getString(Key.TABLE);
List<String> renderedPreSqls = WriterUtil.renderPreOrPostSqls(
preSqls, tableName);
if (renderedPreSqls.size() == 0) {
return;
}
originalConfig.remove(Key.PRE_SQL);
Connection conn = getAdbpgConnect(originalConfig);
WriterUtil.executeSqls(conn, renderedPreSqls, generateJdbcUrl(originalConfig), DATABASE_TYPE);
DBUtil.closeDBResources(null, null, conn);
}
public static void post(Configuration configuration) {
List<String> postSqls = configuration.getList(Key.POST_SQL,
String.class);
String tableName = configuration.getString(Key.TABLE);
List<String> renderedPostSqls = WriterUtil.renderPreOrPostSqls(
postSqls, tableName);
if (renderedPostSqls.size() == 0) {
return;
}
configuration.remove(Key.POST_SQL);
Connection conn = getAdbpgConnect(configuration);
WriterUtil.executeSqls(conn, renderedPostSqls, generateJdbcUrl(configuration), DATABASE_TYPE);
DBUtil.closeDBResources(null, null, conn);
}
}
================================================
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Constant.java
================================================
package com.alibaba.datax.plugin.writer.adbpgwriter.util;
/**
* @author yuncheng
*/
public class Constant {
public static final int DEFAULT_RETRY_TIMES = 3;
public static final String COLUMN_QUOTE_CHARACTER = "\"";
}
================================================
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Key.java
================================================
package com.alibaba.datax.plugin.writer.adbpgwriter.util;
/**
* @author yuncheng
*/
public class Key {
public final static String COLUMN = "column";
public final static String IS_INSERTINGORE = "insertIgnore";
public final static String HOST = "host";
public final static String PORT = "port";
public final static String DATABASE = "database";
public final static String SCHEMA = "schema";
public final static String EMPTY_AS_NULL = "emptyAsNull";
public final static String IGNORE_INSERT = "ignoreInsert";
public final static String RETRY_CONNECTION_TIME = "retryTimes";
public final static String RETRY_INTERVAL_TIME = "retryIntervalTime";
public final static String COMMIT_SIZE = "commitSize";
public final static String PARALLEL_NUMBER = "parallelNumber";
public final static String SHARED_DATASOURCE = "shareDataSource";
}
================================================
FILE: adbpgwriter/src/main/resources/plugin.json
================================================
{
"name": "adbpgwriter",
"class": "com.alibaba.datax.plugin.writer.adbpgwriter.AdbpgWriter",
"description": "",
"developer": "alibaba"
}
================================================
FILE: adbpgwriter/src/main/resources/plugin_job_template.json
================================================
{
"name": "adbpgwriter",
"parameter": {
"username": "",
"password": "",
"host": "",
"port": "",
"database": "",
"schema": "",
"table": "",
"column": ["*"]
}
}
================================================
FILE: adswriter/doc/adswriter.md
================================================
# DataX ADS写入
---
## 1 快速介绍
<br />
欢迎ADS加入DataX生态圈！ADSWriter插件实现了其他数据源向ADS写入功能，现有DataX所有的数据源均可以无缝接入ADS，实现数据快速导入ADS。
ADS写入预计支持两种实现方式：
* ADSWriter 支持向ODPS中转落地导入ADS方式，优点在于当数据量较大时(>1KW)，可以以较快速度进行导入，缺点引入了ODPS作为落地中转，因此牵涉三方系统(DataX、ADS、ODPS)鉴权认证。
* ADSWriter 同时支持向ADS直接写入的方式，优点在于小批量数据写入能够较快完成(<1KW)，缺点在于大数据导入较慢。
注意：
> 如果从ODPS导入数据到ADS，请用户提前在源ODPS的Project中授权ADS Build账号具有读取你源表ODPS的权限，同时，ODPS源表创建人和ADS写入属于同一个阿里云账号。
-
> 如果从非ODPS导入数据到ADS，请用户提前在目的端ADS空间授权ADS Build账号具备Load data权限。
以上涉及ADS Build账号请联系ADS管理员提供。
## 2 实现原理
ADS写入预计支持两种实现方式：
### 2.1 Load模式
DataX 将数据导入ADS为当前导入任务分配的ADS项目表，随后DataX通知ADS完成数据加载。该类数据导入方式实际上是写ADS完成数据同步，由于ADS是分布式存储集群，因此该通道吞吐量较大，可以支持TB级别数据导入。
1. DataX底层得到明文的 jdbc://host:port/dbname + username + password + table， 以此连接ADS， 执行show grants; 前置检查该用户是否有ADS中目标表的Load Data或者更高的权限。注意，此时ADSWriter使用用户填写的ADS用户名+密码信息完成登录鉴权工作。
2. 检查通过后，通过ADS中目标表的元数据反向生成ODPS DDL，在ODPS中间project中，以ADSWriter的账户建立ODPS表（非分区表，生命周期设为1-2Day), 并调用ODPSWriter把数据源的数据写入该ODPS表中。
注意，这里需要使用中转ODPS的账号AK向中转ODPS写入数据。
3. 写入完成后，以中转ODPS账号连接ADS，发起Load Data From ‘odps://中转project/中转table/' [overwrite] into adsdb.adstable [partition (xx,xx=xx)]; 这个命令返回一个Job ID需要记录。
注意，此时ADS使用自己的Build账号访问中转ODPS，因此需要中转ODPS对这个Build账号提前开放读取权限。
4. 连接ADS一分钟一次轮询执行 select state from information_schema.job_instances where job_id like ‘$Job ID’，查询状态，注意这个第一个一分钟可能查不到状态记录。
5. Success或者Fail后返回给用户，然后删除中转ODPS表，任务结束。
上述流程是从其他非ODPS数据源导入ADS流程，对于ODPS导入ADS流程使用如下流程：
### 2.2 Insert模式
DataX 将数据直连ADS接口，利用ADS暴露的INSERT接口直写到ADS。该类数据导入方式写入吞吐量较小，不适合大批量数据写入。有如下注意点：
* ADSWriter使用JDBC连接直连ADS，并只使用了JDBC Statement进行数据插入。ADS不支持PreparedStatement，故ADSWriter只能单行多线程进行写入。
* ADSWriter支持筛选部分列，列换序等功能，即用户可以填写列。
* 考虑到ADS负载问题，建议ADSWriter Insert模式建议用户使用TPS限流，最高在1W TPS。
* ADSWriter在所有Task完成写入任务后，Job Post单例执行flush工作，保证数据在ADS整体更新。
## 3 功能说明
### 3.1 配置样例
* 这里使用一份从内存产生到ADS，使用Load模式进行导入的数据。
```
{
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "streamreader",
"parameter": {
"column": [
{
"value": "DataX",
"type": "string"
},
{
"value": "test",
"type": "bytes"
}
],
"sliceRecordCount": 100000
}
},
"writer": {
"name": "adswriter",
"parameter": {
"odps": {
"accessId": "xxx",
"accessKey": "xxx",
"account": "xxx@aliyun.com",
"odpsServer": "xxx",
"tunnelServer": "xxx",
"project": "transfer_project"
},
"writeMode": "load",
"url": "127.0.0.1:3306",
"schema": "schema",
"table": "table",
"username": "username",
"password": "password",
"partition": "",
"lifeCycle": 2,
"overWrite": true,
}
}
}
]
}
}
```
* 这里使用一份从内存产生到ADS，使用Insert模式进行导入的数据。
```
{
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "streamreader",
"parameter": {
"column": [
{
"value": "DataX",
"type": "string"
},
{
"value": "test",
"type": "bytes"
}
],
"sliceRecordCount": 100000
}
},
"writer": {
"name": "adswriter",
"parameter": {
"writeMode": "insert",
"url": "127.0.0.1:3306",
"schema": "schema",
"table": "table",
"column": ["*"],
"username": "username",
"password": "password",
"partition": "id,ds=2015"
}
}
}
]
}
}
```
### 3.2 参数说明 （用户配置规格）
* **url**
* 描述：ADS连接信息，格式为"ip:port"。
* 必选：是 <br />
* 默认值：无 <br />
* **schema**
* 描述：ADS的schema名称。
* 必选：是 <br />
* 默认值：无 <br />
* **username**
* 描述：ADS对应的username，目前就是accessId <br />
* 必选：是 <br />
* 默认值：无 <br />
* **password**
* 描述：ADS对应的password，目前就是accessKey <br />
* 必选：是 <br />
* 默认值：无 <br />
* **table**
* 描述：目的表的表名称。
* 必选：是 <br />
* 默认值：无 <br />
* **partition**
* 描述：目标表的分区名称，当目标表为分区表，需要指定该字段。
* 必选：否 <br />
* 默认值：无 <br />
* **writeMode**
* 描述：支持Load和Insert两种写入模式
* 必选：是 <br />
* 默认值：无 <br />
* **column**
* 描述：目的表字段列表，可以为["*"]，或者具体的字段列表，例如["a", "b", "c"]
* 必选：是 <br />
* 默认值：无 <br />
* **overWrite**
* 描述：ADS写入是否覆盖当前写入的表，true为覆盖写入，false为不覆盖(追加)写入。当writeMode为Load，该值才会生效。
* 必选：是 <br />
* 默认值：无 <br />
* **lifeCycle**
* 描述：ADS 临时表生命周期。当writeMode为Load时，该值才会生效。
* 必选：是 <br />
* 默认值：无 <br />
* **batchSize**
* 描述：ADS 提交数据写的批量条数，当writeMode为insert时，该值才会生效。
* 必选：writeMode为insert时才有用 <br />
* 默认值：32 <br />
* **bufferSize**
* 描述：DataX数据收集缓冲区大小，缓冲区的目的是攒一个较大的buffer，源头的数据首先进入到此buffer中进行排序，排序完成后再提交ads写。排序是根据ads的分区列模式进行的，排序的目的是数据顺序对ADS服务端更友好，出于性能考虑。bufferSize缓冲区中的数据会经过batchSize批量提交到ADS中，一般如果要设置bufferSize，设置bufferSize为batchSize数量的多倍。当writeMode为insert时，该值才会生效。
* 必选：writeMode为insert时才有用 <br />
* 默认值：默认不配置不开启此功能 <br />
### 3.3 类型转换
| DataX 内部类型| ADS 数据类型 |
| -------- | ----- |
| Long |int, tinyint, smallint, int, bigint|
| Double |float, double, decimal|
| String |varchar |
| Date |date |
| Boolean |bool |
| Bytes |无 |
注意:
* multivalue ADS支持multivalue类型，DataX对于该类型支持待定？
## 4 插件约束
如果Reader为ODPS，且ADSWriter写入模式为Load模式时，ODPS的partition只支持如下三种配置方式(以两级分区为例)：
```
"partition":["pt=*,ds=*"] (读取test表所有分区的数据)
"partition":["pt=1,ds=*"] (读取test表下面，一级分区pt=1下面的所有二级分区)
"partition":["pt=1,ds=hangzhou"] (读取test表下面，一级分区pt=1下面，二级分区ds=hz的数据)
```
## 5 性能报告（线上环境实测）
### 5.1 环境准备
### 5.2 测试报告
## 6 FAQ
================================================
FILE: adswriter/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-all</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>adswriter</artifactId>
<name>adswriter</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-common</artifactId>
<version>${datax-project-version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-core</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>plugin-rdbms-util</artifactId>
<version>${datax-project-version}</version>
<exclusions>
<exclusion>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba.cloud.analyticdb</groupId>
<artifactId>adbclient</artifactId>
<version>1.0.2</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.1.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>odpswriter</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.31</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
</dependencies>
<build>
<plugins>
<!-- compiler plugin -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${jdk-version}</source>
<target>${jdk-version}</target>
<encoding>${project-sourceEncoding}</encoding>
</configuration>
</plugin>
<!-- assembly plugin -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assembly/package.xml</descriptor>
</descriptors>
<finalName>datax</finalName>
</configuration>
<executions>
<execution>
<id>dwzip</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: adswriter/src/main/assembly/package.xml
================================================
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id></id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>plugin.json</include>
<include>config.properties</include>
<include>plugin_job_template.json</include>
</includes>
<outputDirectory>plugin/writer/adswriter</outputDirectory>
</fileSet>
<fileSet>
<directory>target/</directory>
<includes>
<include>adswriter-0.0.1-SNAPSHOT.jar</include>
</includes>
<outputDirectory>plugin/writer/adswriter</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>plugin/writer/adswriter/libs</outputDirectory>
<scope>runtime</scope>
</dependencySet>
</dependencySets>
</assembly>
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsException.java
================================================
package com.alibaba.datax.plugin.writer.adswriter;
public class AdsException extends Exception {
private static final long serialVersionUID = 1080618043484079794L;
public final static int ADS_CONN_URL_NOT_SET = -100;
public final static int ADS_CONN_USERNAME_NOT_SET = -101;
public final static int ADS_CONN_PASSWORD_NOT_SET = -102;
public final static int ADS_CONN_SCHEMA_NOT_SET = -103;
public final static int JOB_NOT_EXIST = -200;
public final static int JOB_FAILED = -201;
public final static int ADS_LOADDATA_SCHEMA_NULL = -300;
public final static int ADS_LOADDATA_TABLE_NULL = -301;
public final static int ADS_LOADDATA_SOURCEPATH_NULL = -302;
public final static int ADS_LOADDATA_JOBID_NOT_AVAIL = -303;
public final static int ADS_LOADDATA_FAILED = -304;
public final static int ADS_TABLEMETA_SCHEMA_NULL = -404;
public final static int ADS_TABLEMETA_TABLE_NULL = -405;
public final static int OTHER = -999;
private int code = OTHER;
private String message;
public AdsException(int code, String message, Throwable e) {
super(message, e);
this.code = code;
this.message = message;
}
@Override
public String getMessage() {
return "Code=" + this.code + " Message=" + this.message;
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsWriter.java
================================================
package com.alibaba.datax.plugin.writer.adswriter;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.TaskPluginCollector;
import com.alibaba.datax.common.spi.Writer;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.rdbms.util.DBUtil;
import com.alibaba.datax.plugin.rdbms.util.DataBaseType;
import com.alibaba.datax.plugin.rdbms.writer.util.WriterUtil;
import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
import com.alibaba.datax.plugin.writer.adswriter.insert.AdsClientProxy;
import com.alibaba.datax.plugin.writer.adswriter.insert.AdsInsertProxy;
import com.alibaba.datax.plugin.writer.adswriter.insert.AdsInsertUtil;
import com.alibaba.datax.plugin.writer.adswriter.insert.AdsProxy;
import com.alibaba.datax.plugin.writer.adswriter.load.AdsHelper;
import com.alibaba.datax.plugin.writer.adswriter.load.TableMetaHelper;
import com.alibaba.datax.plugin.writer.adswriter.load.TransferProjectConf;
import com.alibaba.datax.plugin.writer.adswriter.odps.TableMeta;
import com.alibaba.datax.plugin.writer.adswriter.util.AdsUtil;
import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
import com.alibaba.datax.plugin.writer.adswriter.util.Key;
import com.alibaba.datax.plugin.writer.odpswriter.OdpsWriter;
import com.aliyun.odps.Instance;
import com.aliyun.odps.Odps;
import com.aliyun.odps.OdpsException;
import com.aliyun.odps.account.Account;
import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.task.SQLTask;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class AdsWriter extends Writer {
public static class Job extends Writer.Job {
private static final Logger LOG = LoggerFactory.getLogger(Writer.Job.class);
public final static String ODPS_READER = "odpsreader";
private OdpsWriter.Job odpsWriterJobProxy = new OdpsWriter.Job();
private Configuration originalConfig;
private Configuration readerConfig;
/**
* 持有ads账号的ads helper
*/
private AdsHelper adsHelper;
/**
* 持有odps账号的ads helper
*/
private AdsHelper odpsAdsHelper;
/**
* 中转odps的配置，对应到writer配置的parameter.odps部分
*/
private TransferProjectConf transProjConf;
private final int ODPSOVERTIME = 120000;
private String odpsTransTableName;
private String writeMode;
private long startTime;
@Override
public void init() {
startTime = System.currentTimeMillis();
this.originalConfig = super.getPluginJobConf();
this.writeMode = this.originalConfig.getString(Key.WRITE_MODE);
if(null == this.writeMode) {
LOG.warn("您未指定[writeMode]参数, 默认采用load模式, load模式只能用于离线表");
this.writeMode = Constant.LOADMODE;
this.originalConfig.set(Key.WRITE_MODE, "load");
}
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
AdsUtil.checkNecessaryConfig(this.originalConfig, this.writeMode);
loadModeInit();
} else if(Constant.INSERTMODE.equalsIgnoreCase(this.writeMode) || Constant.STREAMMODE.equalsIgnoreCase(this.writeMode)) {
AdsUtil.checkNecessaryConfig(this.originalConfig, this.writeMode);
List<String> allColumns = AdsInsertUtil.getAdsTableColumnNames(originalConfig);
AdsInsertUtil.dealColumnConf(originalConfig, allColumns);
LOG.debug("After job init(), originalConfig now is:[\n{}\n]",
originalConfig.toJSON());
} else {
throw DataXException.asDataXException(AdsWriterErrorCode.INVALID_CONFIG_VALUE, "writeMode 必须为 'load' 或者 'insert' 或者 'stream'");
}
}
private void loadModeInit() {
this.adsHelper = AdsUtil.createAdsHelper(this.originalConfig);
this.odpsAdsHelper = AdsUtil.createAdsHelperWithOdpsAccount(this.originalConfig);
this.transProjConf = TransferProjectConf.create(this.originalConfig);
// 打印权限申请流程到日志中
LOG.info(String
.format("%s%n%s%n%s",
"如果您直接是odps->ads数据同步, 需要做2方面授权:",
"[1] ads官方账号至少需要有待同步表的describe和select权限, 因为ads系统需要获取odps待同步表的结构和数据信息",
"[2] 您配置的ads数据源访问账号ak, 需要有向指定的ads数据库发起load data的权限, 您可以在ads系统中添加授权"));
LOG.info(String
.format("%s%s%n%s%n%s",
"如果您直接是rds(或其它非odps数据源)->ads数据同步, 流程是先将数据装载如odps临时表，再从odps临时表->ads, ",
String.format("中转odps项目为%s,中转项目账号为%s, 权限方面:",
this.transProjConf.getProject(),
this.transProjConf.getAccount()),
"[1] ads官方账号至少需要有待同步表(这里是odps临时表)的describe和select权限, 因为ads系统需要获取odps待同步表的结构和数据信息，此部分部署时已经完成授权",
String.format("[2] 中转odps对应的账号%s, 需要有向指定的ads数据库发起load data的权限, 您可以在ads系统中添加授权", this.transProjConf.getAccount())));
/**
* 如果是从odps导入到ads，直接load data然后System.exit()
*/
if (super.getPeerPluginName().equals(ODPS_READER)) {
transferFromOdpsAndExit();
}
Account odpsAccount;
odpsAccount = new AliyunAccount(transProjConf.getAccessId(), transProjConf.getAccessKey());
Odps odps = new Odps(odpsAccount);
odps.setEndpoint(transProjConf.getOdpsServer());
odps.setDefaultProject(transProjConf.getProject());
TableMeta tableMeta;
try {
String adsTable = this.originalConfig.getString(Key.ADS_TABLE);
TableInfo tableInfo = adsHelper.getTableInfo(adsTable);
int lifeCycle = this.originalConfig.getInt(Key.Life_CYCLE);
tableMeta = TableMetaHelper.createTempODPSTable(tableInfo, lifeCycle);
this.odpsTransTableName = tableMeta.getTableName();
String sql = tableMeta.toDDL();
LOG.info("正在创建ODPS临时表： "+sql);
Instance instance = SQLTask.run(odps, transProjConf.getProject(), sql, null, null);
boolean terminated = false;
int time = 0;
while (!terminated && time < ODPSOVERTIME) {
Thread.sleep(1000);
terminated = instance.isTerminated();
time += 1000;
}
LOG.info("正在创建ODPS临时表成功");
} catch (AdsException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.ODPS_CREATETABLE_FAILED, e);
}catch (OdpsException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.ODPS_CREATETABLE_FAILED,e);
} catch (InterruptedException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.ODPS_CREATETABLE_FAILED,e);
}
Configuration newConf = AdsUtil.generateConf(this.originalConfig, this.odpsTransTableName,
tableMeta, this.transProjConf);
odpsWriterJobProxy.setPluginJobConf(newConf);
odpsWriterJobProxy.init();
}
/**
* 当reader是odps的时候，直接call ads的load接口，完成后退出。
* 这种情况下，用户在odps reader里头填写的参数只有部分有效。
* 其中accessId、accessKey是忽略掉iao的。
*/
private void transferFromOdpsAndExit() {
this.readerConfig = super.getPeerPluginJobConf();
String odpsTableName = this.readerConfig.getString(Key.ODPSTABLENAME);
List<String> userConfiguredPartitions = this.readerConfig.getList(Key.PARTITION, String.class);
if (userConfiguredPartitions == null) {
userConfiguredPartitions = Collections.emptyList();
}
if(userConfiguredPartitions.size() > 1) {
throw DataXException.asDataXException(AdsWriterErrorCode.ODPS_PARTITION_FAILED, "");
}
if(userConfiguredPartitions.size() == 0) {
loadAdsData(adsHelper, odpsTableName,null);
}else {
loadAdsData(adsHelper, odpsTableName,userConfiguredPartitions.get(0));
}
System.exit(0);
}
// 一般来说，是需要推迟到 task 中进行pre 的执行（单表情况例外）
@Override
public void prepare() {
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
//导数据到odps表中
this.odpsWriterJobProxy.prepare();
} else {
// 实时表模式非分库分表
String adsTable = this.originalConfig.getString(Key.ADS_TABLE);
List<String> preSqls = this.originalConfig.getList(Key.PRE_SQL,
String.class);
List<String> renderedPreSqls = WriterUtil.renderPreOrPostSqls(
preSqls, adsTable);
if (null != renderedPreSqls && !renderedPreSqls.isEmpty()) {
// 说明有 preSql 配置，则此处删除掉
this.originalConfig.remove(Key.PRE_SQL);
Connection preConn = AdsUtil.getAdsConnect(this.originalConfig);
LOG.info("Begin to execute preSqls:[{}]. context info:{}.",
StringUtils.join(renderedPreSqls, ";"),
this.originalConfig.getString(Key.ADS_URL));
WriterUtil.executeSqls(preConn, renderedPreSqls,
this.originalConfig.getString(Key.ADS_URL),
DataBaseType.ADS);
DBUtil.closeDBResources(null, null, preConn);
}
}
}
@Override
public List<Configuration> split(int mandatoryNumber) {
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
return this.odpsWriterJobProxy.split(mandatoryNumber);
} else {
List<Configuration> splitResult = new ArrayList<Configuration>();
for(int i = 0; i < mandatoryNumber; i++) {
splitResult.add(this.originalConfig.clone());
}
return splitResult;
}
}
// 一般来说，是需要推迟到 task 中进行post 的执行（单表情况例外）
@Override
public void post() {
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
loadAdsData(odpsAdsHelper, this.odpsTransTableName, null);
this.odpsWriterJobProxy.post();
} else {
// 实时表模式非分库分表
String adsTable = this.originalConfig.getString(Key.ADS_TABLE);
List<String> postSqls = this.originalConfig.getList(
Key.POST_SQL, String.class);
List<String> renderedPostSqls = WriterUtil.renderPreOrPostSqls(
postSqls, adsTable);
if (null != renderedPostSqls && !renderedPostSqls.isEmpty()) {
// 说明有 preSql 配置，则此处删除掉
this.originalConfig.remove(Key.POST_SQL);
Connection postConn = AdsUtil.getAdsConnect(this.originalConfig);
LOG.info(
"Begin to execute postSqls:[{}]. context info:{}.",
StringUtils.join(renderedPostSqls, ";"),
this.originalConfig.getString(Key.ADS_URL));
WriterUtil.executeSqls(postConn, renderedPostSqls,
this.originalConfig.getString(Key.ADS_URL),
DataBaseType.ADS);
DBUtil.closeDBResources(null, null, postConn);
}
}
}
@Override
public void destroy() {
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
this.odpsWriterJobProxy.destroy();
} else {
//insert mode do noting
}
}
private void loadAdsData(AdsHelper helper, String odpsTableName, String odpsPartition) {
String table = this.originalConfig.getString(Key.ADS_TABLE);
String project;
if (super.getPeerPluginName().equals(ODPS_READER)) {
project = this.readerConfig.getString(Key.PROJECT);
} else {
project = this.transProjConf.getProject();
}
String partition = this.originalConfig.getString(Key.PARTITION);
String sourcePath = AdsUtil.generateSourcePath(project,odpsTableName,odpsPartition);
/**
* 因为之前检查过，所以不用担心unbox的时候NPE
*/
boolean overwrite = this.originalConfig.getBool(Key.OVER_WRITE);
try {
String id = helper.loadData(table,partition,sourcePath,overwrite);
LOG.info("ADS Load Data任务已经提交，job id: " + id);
boolean terminated = false;
int time = 0;
while(!terminated) {
Thread.sleep(120000);
terminated = helper.checkLoadDataJobStatus(id);
time += 2;
LOG.info("ADS 正在导数据中，整个过程需要20分钟以上，请耐心等待,目前已执行 "+ time+" 分钟");
}
LOG.info("ADS 导数据已成功");
} catch (AdsException e) {
if (super.getPeerPluginName().equals(ODPS_READER)) {
// TODO 使用云账号
AdsWriterErrorCode.ADS_LOAD_ODPS_FAILED.setAdsAccount(helper.getUserName());
throw DataXException.asDataXException(AdsWriterErrorCode.ADS_LOAD_ODPS_FAILED,e);
} else {
throw DataXException.asDataXException(AdsWriterErrorCode.ADS_LOAD_TEMP_ODPS_FAILED,e);
}
} catch (InterruptedException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.ODPS_CREATETABLE_FAILED,e);
}
}
}
public static class Task extends Writer.Task {
private static final Logger LOG = LoggerFactory.getLogger(Writer.Task.class);
private Configuration writerSliceConfig;
private OdpsWriter.Task odpsWriterTaskProxy = new OdpsWriter.Task();
private String writeMode;
private String schema;
private String table;
private int columnNumber;
// warn: 只有在insert, stream模式才有, 对于load模式表明为odps临时表了
private TableInfo tableInfo;
private String writeProxy;
AdsProxy proxy = null;
@Override
public void init() {
writerSliceConfig = super.getPluginJobConf();
this.writeMode = this.writerSliceConfig.getString(Key.WRITE_MODE);
this.schema = writerSliceConfig.getString(Key.SCHEMA);
this.table = writerSliceConfig.getString(Key.ADS_TABLE);
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
odpsWriterTaskProxy.setPluginJobConf(writerSliceConfig);
odpsWriterTaskProxy.init();
} else if(Constant.INSERTMODE.equalsIgnoreCase(this.writeMode) || Constant.STREAMMODE.equalsIgnoreCase(this.writeMode)) {
if (Constant.STREAMMODE.equalsIgnoreCase(this.writeMode)) {
this.writeProxy = "datax";
} else {
this.writeProxy = this.writerSliceConfig.getString("writeProxy", "adbClient");
}
this.writerSliceConfig.set("writeProxy", this.writeProxy);
try {
this.tableInfo = AdsUtil.createAdsHelper(this.writerSliceConfig).getTableInfo(this.table);
} catch (AdsException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.CREATE_ADS_HELPER_FAILED, e);
}
List<String> allColumns = new ArrayList<String>();
List<ColumnInfo> columnInfo = this.tableInfo.getColumns();
for (ColumnInfo eachColumn : columnInfo) {
allColumns.add(eachColumn.getName());
}
LOG.info("table:[{}] all columns:[\n{}\n].", this.writerSliceConfig.get(Key.ADS_TABLE), StringUtils.join(allColumns, ","));
AdsInsertUtil.dealColumnConf(writerSliceConfig, allColumns);
List<String> userColumns = writerSliceConfig.getList(Key.COLUMN, String.class);
this.columnNumber = userColumns.size();
} else {
throw DataXException.asDataXException(AdsWriterErrorCode.INVALID_CONFIG_VALUE, "writeMode 必须为 'load' 或者 'insert' 或者 'stream'");
}
}
@Override
public void prepare() {
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
odpsWriterTaskProxy.prepare();
} else {
//do nothing
}
}
public void startWrite(RecordReceiver recordReceiver) {
// 这里的是非odps数据源->odps中转临时表数据同步, load操作在job post阶段完成
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
odpsWriterTaskProxy.setTaskPluginCollector(super.getTaskPluginCollector());
odpsWriterTaskProxy.startWrite(recordReceiver);
} else {
// insert 模式
List<String> columns = writerSliceConfig.getList(Key.COLUMN, String.class);
Connection connection = AdsUtil.getAdsConnect(this.writerSliceConfig);
TaskPluginCollector taskPluginCollector = super.getTaskPluginCollector();
if (StringUtils.equalsIgnoreCase(this.writeProxy, "adbClient")) {
this.proxy = new AdsClientProxy(table, columns, writerSliceConfig, taskPluginCollector, this.tableInfo);
} else {
this.proxy = new AdsInsertProxy(schema + "." + table, columns, writerSliceConfig, taskPluginCollector, this.tableInfo);
}
proxy.startWriteWithConnection(recordReceiver, connection, columnNumber);
}
}
@Override
public void post() {
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
odpsWriterTaskProxy.post();
} else {
//do noting until now
}
}
@Override
public void destroy() {
if(Constant.LOADMODE.equalsIgnoreCase(this.writeMode)) {
odpsWriterTaskProxy.destroy();
} else {
//do noting until now
if (null != this.proxy) {
this.proxy.closeResource();
}
}
}
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsWriterErrorCode.java
================================================
package com.alibaba.datax.plugin.writer.adswriter;
import com.alibaba.datax.common.spi.ErrorCode;
public enum AdsWriterErrorCode implements ErrorCode {
REQUIRED_VALUE("AdsWriter-00", "您缺失了必须填写的参数值."),
NO_ADS_TABLE("AdsWriter-01", "ADS表不存在."),
ODPS_CREATETABLE_FAILED("AdsWriter-02", "创建ODPS临时表失败，请联系ADS 技术支持"),
ADS_LOAD_TEMP_ODPS_FAILED("AdsWriter-03", "ADS从ODPS临时表导数据失败，请联系ADS 技术支持"),
TABLE_TRUNCATE_ERROR("AdsWriter-04", "清空 ODPS 目的表时出错."),
CREATE_ADS_HELPER_FAILED("AdsWriter-05", "创建ADSHelper对象出错，请联系ADS 技术支持"),
ODPS_PARTITION_FAILED("AdsWriter-06", "ODPS Reader不允许配置多个partition，目前只支持三种配置方式，\"partition\":[\"pt=*,ds=*\"](读取test表所有分区的数据)； \n" +
"\"partition\":[\"pt=1,ds=*\"](读取test表下面，一级分区pt=1下面的所有二级分区)； \n" +
"\"partition\":[\"pt=1,ds=hangzhou\"](读取test表下面，一级分区pt=1下面，二级分区ds=hz的数据)"),
ADS_LOAD_ODPS_FAILED("AdsWriter-07", "ADS从ODPS导数据失败，请联系ADS 技术支持，先检查ADS账号是否已加到该ODPS Project中。ADS账号为："),
INVALID_CONFIG_VALUE("AdsWriter-08", "不合法的配置值."),
GET_ADS_TABLE_MEATA_FAILED("AdsWriter-11", "获取ADS table原信息失败");
private final String code;
private final String description;
private String adsAccount;
private AdsWriterErrorCode(String code, String description) {
this.code = code;
this.description = description;
}
public void setAdsAccount(String adsAccount) {
this.adsAccount = adsAccount;
}
@Override
public String getCode() {
return this.code;
}
@Override
public String getDescription() {
return this.description;
}
@Override
public String toString() {
if (this.code.equals("AdsWriter-07")){
return String.format("Code:[%s], Description:[%s][%s]. ", this.code,
this.description,adsAccount);
}else{
return String.format("Code:[%s], Description:[%s]. ", this.code,
this.description);
}
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnDataType.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.ads;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
/**
* ADS column data type.
*
* @since 0.0.1
*/
public class ColumnDataType {
// public static final int NULL = 0;
public static final int BOOLEAN = 1;
public static final int BYTE = 2;
public static final int SHORT = 3;
public static final int INT = 4;
public static final int LONG = 5;
public static final int DECIMAL = 6;
public static final int DOUBLE = 7;
public static final int FLOAT = 8;
public static final int TIME = 9;
public static final int DATE = 10;
public static final int TIMESTAMP = 11;
public static final int STRING = 13;
// public static final int STRING_IGNORECASE = 14;
// public static final int STRING_FIXED = 21;
public static final int MULTI_VALUE = 22;
public static final int TYPE_COUNT = MULTI_VALUE + 1;
/**
* The list of types. An ArrayList so that Tomcat doesn't set it to null when clearing references.
*/
private static final ArrayList<ColumnDataType> TYPES = new ArrayList<ColumnDataType>();
private static final HashMap<String, ColumnDataType> TYPES_BY_NAME = new HashMap<String, ColumnDataType>();
private static final ArrayList<ColumnDataType> TYPES_BY_VALUE_TYPE = new ArrayList<ColumnDataType>();
/**
* @param dataTypes
* @return
*/
public static String getNames(int[] dataTypes) {
List<String> names = new ArrayList<String>(dataTypes.length);
for (final int dataType : dataTypes) {
names.add(ColumnDataType.getDataType(dataType).name);
}
return names.toString();
}
public int type;
public String name;
public int sqlType;
public String jdbc;
/**
* How closely the data type maps to the corresponding JDBC SQL type (low is best).
*/
public int sqlTypePos;
static {
for (int i = 0; i < TYPE_COUNT; i++) {
TYPES_BY_VALUE_TYPE.add(null);
}
// add(NULL, Types.NULL, "Null", new String[] { "NULL" });
add(STRING, Types.VARCHAR, "String", new String[] { "VARCHAR", "VARCHAR2", "NVARCHAR", "NVARCHAR2",
"VARCHAR_CASESENSITIVE", "CHARACTER VARYING", "TID" });
add(STRING, Types.LONGVARCHAR, "String", new String[] { "LONGVARCHAR", "LONGNVARCHAR" });
// add(STRING_FIXED, Types.CHAR, "String", new String[] { "CHAR", "CHARACTER", "NCHAR" });
// add(STRING_IGNORECASE, Types.VARCHAR, "String", new String[] { "VARCHAR_IGNORECASE" });
add(BOOLEAN, Types.BOOLEAN, "Boolean", new String[] { "BOOLEAN", "BIT", "BOOL" });
add(BYTE, Types.TINYINT, "Byte", new String[] { "TINYINT" });
add(SHORT, Types.SMALLINT, "Short", new String[] { "SMALLINT", "YEAR", "INT2" });
add(INT, Types.INTEGER, "Int", new String[] { "INTEGER", "INT", "MEDIUMINT", "INT4", "SIGNED" });
add(INT, Types.INTEGER, "Int", new String[] { "SERIAL" });
add(LONG, Types.BIGINT, "Long", new String[] { "BIGINT", "INT8", "LONG" });
add(LONG, Types.BIGINT, "Long", new String[] { "IDENTITY", "BIGSERIAL" });
add(DECIMAL, Types.DECIMAL, "BigDecimal", new String[] { "DECIMAL", "DEC" });
add(DECIMAL, Types.NUMERIC, "BigDecimal", new String[] { "NUMERIC", "NUMBER" });
add(FLOAT, Types.REAL, "Float", new String[] { "REAL", "FLOAT4" });
add(DOUBLE, Types.DOUBLE, "Double", new String[] { "DOUBLE", "DOUBLE PRECISION" });
add(DOUBLE, Types.FLOAT, "Double", new String[] { "FLOAT", "FLOAT8" });
add(TIME, Types.TIME, "Time", new String[] { "TIME" });
add(DATE, Types.DATE, "Date", new String[] { "DATE" });
add(TIMESTAMP, Types.TIMESTAMP, "Timestamp", new String[] { "TIMESTAMP", "DATETIME", "SMALLDATETIME" });
add(MULTI_VALUE, Types.VARCHAR, "String", new String[] { "MULTIVALUE" });
}
private static void add(int type, int sqlType, String jdbc, String[] names) {
for (int i = 0; i < names.length; i++) {
ColumnDataType dt = new ColumnDataType();
dt.type = type;
dt.sqlType = sqlType;
dt.jdbc = jdbc;
dt.name = names[i];
for (ColumnDataType t2 : TYPES) {
if (t2.sqlType == dt.sqlType) {
dt.sqlTypePos++;
}
}
TYPES_BY_NAME.put(dt.name, dt);
if (TYPES_BY_VALUE_TYPE.get(type) == null) {
TYPES_BY_VALUE_TYPE.set(type, dt);
}
TYPES.add(dt);
}
}
/**
* Get the list of data types.
*
* @return the list
*/
public static ArrayList<ColumnDataType> getTypes() {
return TYPES;
}
/**
* Get the name of the Java class for the given value type.
*
* @param type the value type
* @return the class name
*/
public static String getTypeClassName(int type) {
switch (type) {
case BOOLEAN:
// "java.lang.Boolean";
return Boolean.class.getName();
case BYTE:
// "java.lang.Byte";
return Byte.class.getName();
case SHORT:
// "java.lang.Short";
return Short.class.getName();
case INT:
// "java.lang.Integer";
return Integer.class.getName();
case LONG:
// "java.lang.Long";
return Long.class.getName();
case DECIMAL:
// "java.math.BigDecimal";
return BigDecimal.class.getName();
case TIME:
// "java.sql.Time";
return Time.class.getName();
case DATE:
// "java.sql.Date";
return Date.class.getName();
case TIMESTAMP:
// "java.sql.Timestamp";
return Timestamp.class.getName();
case STRING:
// case STRING_IGNORECASE:
// case STRING_FIXED:
case MULTI_VALUE:
// "java.lang.String";
return String.class.getName();
case DOUBLE:
// "java.lang.Double";
return Double.class.getName();
case FLOAT:
// "java.lang.Float";
return Float.class.getName();
// case NULL:
// return null;
default:
throw new IllegalArgumentException("type=" + type);
}
}
/**
* Get the data type object for the given value type.
*
* @param type the value type
* @return the data type object
*/
public static ColumnDataType getDataType(int type) {
if (type < 0 || type >= TYPE_COUNT) {
throw new IllegalArgumentException("type=" + type);
}
ColumnDataType dt = TYPES_BY_VALUE_TYPE.get(type);
// if (dt == null) {
// dt = TYPES_BY_VALUE_TYPE.get(NULL);
// }
return dt;
}
/**
* Convert a value type to a SQL type.
*
* @param type the value type
* @return the SQL type
*/
public static int convertTypeToSQLType(int type) {
return getDataType(type).sqlType;
}
/**
* Convert a SQL type to a value type.
*
* @param sqlType the SQL type
* @return the value type
*/
public static int convertSQLTypeToValueType(int sqlType) {
switch (sqlType) {
// case Types.CHAR:
// case Types.NCHAR:
// return STRING_FIXED;
case Types.VARCHAR:
case Types.LONGVARCHAR:
case Types.NVARCHAR:
case Types.LONGNVARCHAR:
return STRING;
case Types.NUMERIC:
case Types.DECIMAL:
return DECIMAL;
case Types.BIT:
case Types.BOOLEAN:
return BOOLEAN;
case Types.INTEGER:
return INT;
case Types.SMALLINT:
return SHORT;
case Types.TINYINT:
return BYTE;
case Types.BIGINT:
return LONG;
case Types.REAL:
return FLOAT;
case Types.DOUBLE:
case Types.FLOAT:
return DOUBLE;
case Types.DATE:
return DATE;
case Types.TIME:
return TIME;
case Types.TIMESTAMP:
return TIMESTAMP;
// case Types.NULL:
// return NULL;
default:
throw new IllegalArgumentException("JDBC Type: " + sqlType);
}
}
/**
* Get the value type for the given Java class.
*
* @param x the Java class
* @return the value type
*/
public static int getTypeFromClass(Class<?> x) {
// if (x == null || Void.TYPE == x) {
// return NULL;
// }
if (x.isPrimitive()) {
x = getNonPrimitiveClass(x);
}
if (String.class == x) {
return STRING;
} else if (Integer.class == x) {
return INT;
} else if (Long.class == x) {
return LONG;
} else if (Boolean.class == x) {
return BOOLEAN;
} else if (Double.class == x) {
return DOUBLE;
} else if (Byte.class == x) {
return BYTE;
} else if (Short.class == x) {
return SHORT;
} else if (Float.class == x) {
return FLOAT;
// } else if (Void.class == x) {
// return NULL;
} else if (BigDecimal.class.isAssignableFrom(x)) {
return DECIMAL;
} else if (Date.class.isAssignableFrom(x)) {
return DATE;
} else if (Time.class.isAssignableFrom(x)) {
return TIME;
} else if (Timestamp.class.isAssignableFrom(x)) {
return TIMESTAMP;
} else if (java.util.Date.class.isAssignableFrom(x)) {
return TIMESTAMP;
} else {
throw new IllegalArgumentException("class=" + x);
}
}
/**
* Convert primitive class names to java.lang.* class names.
*
* @param clazz the class (for example: int)
* @return the non-primitive class (for example: java.lang.Integer)
*/
public static Class<?> getNonPrimitiveClass(Class<?> clazz) {
if (!clazz.isPrimitive()) {
return clazz;
} else if (clazz == boolean.class) {
return Boolean.class;
} else if (clazz == byte.class) {
return Byte.class;
} else if (clazz == char.class) {
return Character.class;
} else if (clazz == double.class) {
return Double.class;
} else if (clazz == float.class) {
return Float.class;
} else if (clazz == int.class) {
return Integer.class;
} else if (clazz == long.class) {
return Long.class;
} else if (clazz == short.class) {
return Short.class;
} else if (clazz == void.class) {
return Void.class;
}
return clazz;
}
/**
* Get a data type object from a type name.
*
* @param s the type name
* @return the data type object
*/
public static ColumnDataType getTypeByName(String s) {
return TYPES_BY_NAME.get(s);
}
/**
* Check if the given value type is a String (VARCHAR,...).
*
* @param type the value type
* @return true if the value type is a String type
*/
public static boolean isStringType(int type) {
if (type == STRING /* || type == STRING_FIXED || type == STRING_IGNORECASE */
|| type == MULTI_VALUE) {
return true;
}
return false;
}
/**
* @return
*/
public boolean supportsAdd() {
return supportsAdd(type);
}
/**
* Check if the given value type supports the add operation.
*
* @param type the value type
* @return true if add is supported
*/
public static boolean supportsAdd(int type) {
switch (type) {
case BYTE:
case DECIMAL:
case DOUBLE:
case FLOAT:
case INT:
case LONG:
case SHORT:
return true;
default:
return false;
}
}
/**
* Get the data type that will not overflow when calling 'add' 2 billion times.
*
* @param type the value type
* @return the data type that supports adding
*/
public static int getAddProofType(int type) {
switch (type) {
case BYTE:
return LONG;
case FLOAT:
return DOUBLE;
case INT:
return LONG;
case LONG:
return DECIMAL;
case SHORT:
return LONG;
default:
return type;
}
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.ads;
/**
* ADS column meta.<br>
* <p>
* select ordinal_position,column_name,data_type,type_name,column_comment <br>
* from information_schema.columns <br>
* where table_schema='db_name' and table_name='table_name' <br>
* and is_deleted=0 <br>
* order by ordinal_position limit 1000 <br>
* </p>
*
* @since 0.0.1
*/
public class ColumnInfo {
private int ordinal;
private String name;
private ColumnDataType dataType;
private boolean isDeleted;
private String comment;
public int getOrdinal() {
return ordinal;
}
public void setOrdinal(int ordinal) {
this.ordinal = ordinal;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public ColumnDataType getDataType() {
return dataType;
}
public void setDataType(ColumnDataType dataType) {
this.dataType = dataType;
}
public boolean isDeleted() {
return isDeleted;
}
public void setDeleted(boolean isDeleted) {
this.isDeleted = isDeleted;
}
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("ColumnInfo [ordinal=").append(ordinal).append(", name=").append(name).append(", dataType=")
.append(dataType).append(", isDeleted=").append(isDeleted).append(", comment=").append(comment)
.append("]");
return builder.toString();
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.ads;
import java.util.ArrayList;
import java.util.List;
/**
* ADS table meta.<br>
* <p>
* select table_schema, table_name,comments <br>
* from information_schema.tables <br>
* where table_schema='alimama' and table_name='click_af' limit 1 <br>
* </p>
* <p>
* select ordinal_position,column_name,data_type,type_name,column_comment <br>
* from information_schema.columns <br>
* where table_schema='db_name' and table_name='table_name' <br>
* and is_deleted=0 <br>
* order by ordinal_position limit 1000 <br>
* </p>
*
* @since 0.0.1
*/
public class TableInfo {
private String tableSchema;
private String tableName;
private List<ColumnInfo> columns;
private String comments;
private String tableType;
private String updateType;
private String partitionType;
private String partitionColumn;
private int partitionCount;
private List<String> primaryKeyColumns;
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("TableInfo [tableSchema=").append(tableSchema).append(", tableName=").append(tableName)
.append(", columns=").append(columns).append(", comments=").append(comments).append(",updateType=").append(updateType)
.append(",partitionType=").append(partitionType).append(",partitionColumn=").append(partitionColumn).append(",partitionCount=").append(partitionCount)
.append(",primaryKeyColumns=").append(primaryKeyColumns).append("]");
return builder.toString();
}
public String getTableSchema() {
return tableSchema;
}
public void setTableSchema(String tableSchema) {
this.tableSchema = tableSchema;
}
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public List<ColumnInfo> getColumns() {
return columns;
}
public List<String> getColumnsNames() {
List<String> columnNames = new ArrayList<String>();
for (ColumnInfo column : this.getColumns()) {
columnNames.add(column.getName());
}
return columnNames;
}
public void setColumns(List<ColumnInfo> columns) {
this.columns = columns;
}
public String getComments() {
return comments;
}
public void setComments(String comments) {
this.comments = comments;
}
public String getTableType() {
return tableType;
}
public void setTableType(String tableType) {
this.tableType = tableType;
}
public String getUpdateType() {
return updateType;
}
public void setUpdateType(String updateType) {
this.updateType = updateType;
}
public String getPartitionType() {
return partitionType;
}
public void setPartitionType(String partitionType) {
this.partitionType = partitionType;
}
public String getPartitionColumn() {
return partitionColumn;
}
public void setPartitionColumn(String partitionColumn) {
this.partitionColumn = partitionColumn;
}
public int getPartitionCount() {
return partitionCount;
}
public void setPartitionCount(int partitionCount) {
this.partitionCount = partitionCount;
}
public List<String> getPrimaryKeyColumns() {
return primaryKeyColumns;
}
public void setPrimaryKeyColumns(List<String> primaryKeyColumns) {
this.primaryKeyColumns = primaryKeyColumns;
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java
================================================
/**
* ADS meta and service.
*
* @since 0.0.1
*/
package com.alibaba.datax.plugin.writer.adswriter.ads;
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsClientProxy.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.insert;
import com.alibaba.cloud.analyticdb.adbclient.AdbClient;
import com.alibaba.cloud.analyticdb.adbclient.AdbClientException;
import com.alibaba.cloud.analyticdb.adbclient.DatabaseConfig;
import com.alibaba.cloud.analyticdb.adbclient.Row;
import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.element.StringColumn;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.TaskPluginCollector;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.core.transport.record.DefaultRecord;
import com.alibaba.datax.plugin.rdbms.util.DBUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
import com.alibaba.datax.plugin.writer.adswriter.util.Key;
import com.alibaba.fastjson2.JSON;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.Types;
import java.util.*;
public class AdsClientProxy implements AdsProxy {
private static final Logger LOG = LoggerFactory.getLogger(AdsClientProxy.class);
private String table;
private TaskPluginCollector taskPluginCollector;
public Configuration configuration;
// columnName: <java sql type, ads type name>
private Map<String, Pair<Integer, String>> adsTableColumnsMetaData;
private Map<String, Pair<Integer, String>> userConfigColumnsMetaData;
private boolean useRawData[];
private AdbClient adbClient;
/**
* warn: not support columns as *
*/
public AdsClientProxy(String table, List<String> columns, Configuration configuration,
TaskPluginCollector taskPluginCollector, TableInfo tableInfo) {
this.configuration = configuration;
this.taskPluginCollector = taskPluginCollector;
this.adsTableColumnsMetaData = AdsInsertUtil.getColumnMetaData(tableInfo, columns);
this.userConfigColumnsMetaData = new HashMap<String, Pair<Integer, String>>();
List<String> adsColumnsNames = tableInfo.getColumnsNames();
// 要使用用户配置的column顺序
this.useRawData = new boolean[columns.size()];
for (int i = 0; i < columns.size(); i++) {
String oriEachColumn = columns.get(i);
String eachColumn = oriEachColumn;
// 防御性保留字
if (eachColumn.startsWith(Constant.ADS_QUOTE_CHARACTER)
&& eachColumn.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
eachColumn = eachColumn.substring(1, eachColumn.length() - 1);
}
for (String eachAdsColumn : adsColumnsNames) {
if (eachColumn.equalsIgnoreCase(eachAdsColumn)) {
Pair<Integer, String> eachMeta = this.adsTableColumnsMetaData.get(eachAdsColumn);
this.userConfigColumnsMetaData.put(oriEachColumn, eachMeta);
int columnSqltype = eachMeta.getLeft();
switch (columnSqltype) {
case Types.DATE:
case Types.TIME:
case Types.TIMESTAMP:
this.useRawData[i] = false;
break;
default:
this.useRawData[i] = true;
break;
}
}
}
}
DatabaseConfig databaseConfig = new DatabaseConfig();
String url = configuration.getString(Key.ADS_URL);
String[] hostAndPort = StringUtils.split(url, ":");
if (hostAndPort.length != 2) {
throw DataXException.asDataXException(AdsWriterErrorCode.INVALID_CONFIG_VALUE,
"url should be in host:port format!");
}
this.table = table.toLowerCase();
databaseConfig.setHost(hostAndPort[0]);
databaseConfig.setPort(Integer.parseInt(hostAndPort[1]));
databaseConfig.setUser(configuration.getString(Key.USERNAME));
databaseConfig.setPassword(configuration.getString(Key.PASSWORD));
databaseConfig.setDatabase(configuration.getString(Key.SCHEMA));
databaseConfig.setTable(Collections.singletonList(this.table));
databaseConfig.setColumns(this.table, columns);
// 如果出现insert失败，是否跳过
boolean ignoreInsertError = configuration.getBool("ignoreInsertError", false);
databaseConfig.setIgnoreInsertError(ignoreInsertError);
// If the value of column is empty, set null
boolean emptyAsNull = configuration.getBool(Key.EMPTY_AS_NULL, false);
databaseConfig.setEmptyAsNull(emptyAsNull);
// 使用insert ignore into方式进行插入
boolean ignoreInsert = configuration.getBool(Key.IGNORE_INSERT, false);
databaseConfig.setInsertIgnore(ignoreInsert);
// commit时，写入ADB出现异常时重试的3次
int retryTimes = configuration.getInt(Key.RETRY_CONNECTION_TIME, Constant.DEFAULT_RETRY_TIMES);
databaseConfig.setRetryTimes(retryTimes);
// 重试间隔的时间为1s，单位是ms
int retryIntervalTime = configuration.getInt(Key.RETRY_INTERVAL_TIME, 1000);
databaseConfig.setRetryIntervalTime(retryIntervalTime);
// 设置自动提交的SQL长度（单位Byte），默认为32KB，一般不建议设置
int commitSize = configuration.getInt("commitSize", 32768);
databaseConfig.setCommitSize(commitSize);
// sdk默认为true
boolean partitionBatch = configuration.getBool("partitionBatch", true);
databaseConfig.setPartitionBatch(partitionBatch);
// 设置写入adb时的并发线程数，默认4，针对配置的所有表
int parallelNumber = configuration.getInt("parallelNumber", 4);
databaseConfig.setParallelNumber(parallelNumber);
// 设置client中使用的logger对象，此处使用slf4j.Logger
databaseConfig.setLogger(AdsClientProxy.LOG);
// 设置在拼接insert sql时是否需要带上字段名，默认为true
boolean insertWithColumnName = configuration.getBool("insertWithColumnName", true);
databaseConfig.setInsertWithColumnName(insertWithColumnName);
// sdk 默认值为true
boolean shareDataSource = configuration.getBool("shareDataSource", true);
databaseConfig.setShareDataSource(shareDataSource);
String password = databaseConfig.getPassword();
databaseConfig.setPassword(password.replaceAll(".", "*"));
// 避免敏感信息直接打印
LOG.info("Adb database config is : {}", JSON.toJSONString(databaseConfig));
databaseConfig.setPassword(password);
// Initialize AdbClient，初始化实例之后，databaseConfig的配置信息不能再修改
this.adbClient = new AdbClient(databaseConfig);
}
@Override
public void startWriteWithConnection(RecordReceiver recordReceiver, Connection connection, int columnNumber) {
try {
Record record;
while ((record = recordReceiver.getFromReader()) != null) {
Row row = new Row();
List<Object> values = new ArrayList<Object>();
this.prepareColumnTypeValue(record, values);
row.setColumnValues(values);
try {
this.adbClient.addRow(this.table, row);
} catch (AdbClientException e) {
if (101 == e.getCode()) {
for (String each : e.getErrData()) {
Record dirtyData = new DefaultRecord();
dirtyData.addColumn(new StringColumn(each));
this.taskPluginCollector.collectDirtyRecord(dirtyData, e.getMessage());
}
} else {
throw e;
}
}
}
try {
this.adbClient.commit();
} catch (AdbClientException e) {
if (101 == e.getCode()) {
for (String each : e.getErrData()) {
Record dirtyData = new DefaultRecord();
dirtyData.addColumn(new StringColumn(each));
this.taskPluginCollector.collectDirtyRecord(dirtyData, e.getMessage());
}
} else {
throw e;
}
}
} catch (Exception e) {
throw DataXException.asDataXException(DBUtilErrorCode.WRITE_DATA_ERROR, e);
} finally {
DBUtil.closeDBResources(null, null, connection);
}
}
private void prepareColumnTypeValue(Record record, List<Object> values) {
for (int i = 0; i < this.useRawData.length; i++) {
Column column = record.getColumn(i);
if (this.useRawData[i]) {
values.add(column.getRawData());
} else {
values.add(column.asString());
}
}
}
@Override
public void closeResource() {
try {
LOG.info("stop the adbClient");
this.adbClient.stop();
} catch (Exception e) {
LOG.warn("stop adbClient meet a exception, ignore it: {}", e.getMessage(), e);
}
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.insert;
import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.TaskPluginCollector;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.common.util.RetryUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
import com.alibaba.datax.plugin.writer.adswriter.util.AdsUtil;
import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
import com.alibaba.datax.plugin.writer.adswriter.util.Key;
import com.mysql.jdbc.JDBC4PreparedStatement;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
public class AdsInsertProxy implements AdsProxy {
private static final Logger LOG = LoggerFactory
.getLogger(AdsInsertProxy.class);
private static final boolean IS_DEBUG_ENABLE = LOG.isDebugEnabled();
private static final int MAX_EXCEPTION_CAUSE_ITER = 100;
private String table;
private List<String> columns;
private TaskPluginCollector taskPluginCollector;
private Configuration configuration;
private Boolean emptyAsNull;
private String writeMode;
private String insertSqlPrefix;
private String deleteSqlPrefix;
private int opColumnIndex;
private String lastDmlMode;
// columnName: <java sql type, ads type name>
private Map<String, Pair<Integer, String>> adsTableColumnsMetaData;
private Map<String, Pair<Integer, String>> userConfigColumnsMetaData;
// columnName: index @ ads column
private Map<String, Integer> primaryKeyNameIndexMap;
private int retryTimeUpperLimit;
private Connection currentConnection;
private String partitionColumn;
private int partitionColumnIndex = -1;
private int partitionCount;
public AdsInsertProxy(String table, List<String> columns, Configuration configuration, TaskPluginCollector taskPluginCollector, TableInfo tableInfo) {
this.table = table;
this.columns = columns;
this.configuration = configuration;
this.taskPluginCollector = taskPluginCollector;
this.emptyAsNull = configuration.getBool(Key.EMPTY_AS_NULL, false);
this.writeMode = configuration.getString(Key.WRITE_MODE);
this.insertSqlPrefix = String.format(Constant.INSERT_TEMPLATE, this.table, StringUtils.join(columns, ","));
this.deleteSqlPrefix = String.format(Constant.DELETE_TEMPLATE, this.table);
this.opColumnIndex = configuration.getInt(Key.OPIndex, 0);
this.retryTimeUpperLimit = configuration.getInt(
Key.RETRY_CONNECTION_TIME, Constant.DEFAULT_RETRY_TIMES);
this.partitionCount = tableInfo.getPartitionCount();
this.partitionColumn = tableInfo.getPartitionColumn();
//目前ads新建的表如果未插入数据不能通过select colums from table where 1=2，获取列信息，需要读取ads数据字典
//not this: this.resultSetMetaData = DBUtil.getColumnMetaData(connection, this.table, StringUtils.join(this.columns, ","));
//no retry here(fetch meta data) 注意实时表列换序的可能
this.adsTableColumnsMetaData = AdsInsertUtil.getColumnMetaData(tableInfo, this.columns);
this.userConfigColumnsMetaData = new HashMap<String, Pair<Integer, String>>();
List<String> primaryKeyColumnName = tableInfo.getPrimaryKeyColumns();
List<String> adsColumnsNames = tableInfo.getColumnsNames();
this.primaryKeyNameIndexMap = new HashMap<String, Integer>();
//warn: 要使用用户配置的column顺序, 不要使用从ads元数据获取的column顺序, 原来复用load列顺序其实有问题的
for (int i = 0; i < this.columns.size(); i++) {
String oriEachColumn = this.columns.get(i);
String eachColumn = oriEachColumn;
// 防御性保留字
if (eachColumn.startsWith(Constant.ADS_QUOTE_CHARACTER) && eachColumn.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
eachColumn = eachColumn.substring(1, eachColumn.length() - 1);
}
for (String eachPrimary : primaryKeyColumnName) {
if (eachColumn.equalsIgnoreCase(eachPrimary)) {
this.primaryKeyNameIndexMap.put(oriEachColumn, i);
}
}
for (String eachAdsColumn : adsColumnsNames) {
if (eachColumn.equalsIgnoreCase(eachAdsColumn)) {
this.userConfigColumnsMetaData.put(oriEachColumn, this.adsTableColumnsMetaData.get(eachAdsColumn));
}
}
// 根据第几个column分区列排序，ads实时表只有一级分区、最多256个分区
if (eachColumn.equalsIgnoreCase(this.partitionColumn)) {
this.partitionColumnIndex = i;
}
}
}
public void startWriteWithConnection(RecordReceiver recordReceiver,
Connection connection,
int columnNumber) {
this.currentConnection = connection;
int batchSize = this.configuration.getInt(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE);
// 默认情况下bufferSize需要和batchSize一致
int bufferSize = this.configuration.getInt(Key.BUFFER_SIZE, batchSize);
// insert缓冲，多个分区排序后insert合并发送到ads
List<Record> writeBuffer = new ArrayList<Record>(bufferSize);
List<Record> deleteBuffer = null;
if (this.writeMode.equalsIgnoreCase(Constant.STREAMMODE)) {
// delete缓冲，多个分区排序后delete合并发送到ads
deleteBuffer = new ArrayList<Record>(bufferSize);
}
try {
Record record;
while ((record = recordReceiver.getFromReader()) != null) {
if (this.writeMode.equalsIgnoreCase(Constant.INSERTMODE)) {
if (record.getColumnNumber() != columnNumber) {
// 源头读取字段列数与目的表字段写入列数不相等，直接报错
throw DataXException
.asDataXException(
DBUtilErrorCode.CONF_ERROR,
String.format(
"列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不相等. 请检查您的配置并作出修改.",
record.getColumnNumber(),
columnNumber));
}
writeBuffer.add(record);
if (writeBuffer.size() >= bufferSize) {
this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
writeBuffer.clear();
}
} else {
if (record.getColumnNumber() != columnNumber + 1) {
// 源头读取字段列数需要为目的表字段写入列数+1, 直接报错, 源头多了一列OP
throw DataXException
.asDataXException(
DBUtilErrorCode.CONF_ERROR,
String.format(
"列配置信息有错误. 因为您配置的任务中，源头读取字段数:%s 与 目的表要写入的字段数:%s 不满足源头多1列操作类型列. 请检查您的配置并作出修改.",
record.getColumnNumber(),
columnNumber));
}
String optionColumnValue = record.getColumn(this.opColumnIndex).asString();
OperationType operationType = OperationType.asOperationType(optionColumnValue);
if (operationType.isInsertTemplate()) {
writeBuffer.add(record);
if (this.lastDmlMode == null || this.lastDmlMode == Constant.INSERTMODE) {
this.lastDmlMode = Constant.INSERTMODE;
if (writeBuffer.size() >= bufferSize) {
this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
writeBuffer.clear();
}
} else {
this.lastDmlMode = Constant.INSERTMODE;
// 模式变换触发一次提交ads delete, 并进入insert模式
this.doBatchRecordWithPartitionSort(deleteBuffer, Constant.DELETEMODE, bufferSize, batchSize);
deleteBuffer.clear();
}
} else if (operationType.isDeleteTemplate()) {
deleteBuffer.add(record);
if (this.lastDmlMode == null || this.lastDmlMode == Constant.DELETEMODE) {
this.lastDmlMode = Constant.DELETEMODE;
if (deleteBuffer.size() >= bufferSize) {
this.doBatchRecordWithPartitionSort(deleteBuffer, Constant.DELETEMODE, bufferSize, batchSize);
deleteBuffer.clear();
}
} else {
this.lastDmlMode = Constant.DELETEMODE;
// 模式变换触发一次提交ads insert, 并进入delete模式
this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
writeBuffer.clear();
}
} else {
// 注意OP操作类型的脏数据, 这里不需要重试
this.taskPluginCollector.collectDirtyRecord(record, String.format("不支持您的更新类型:%s", optionColumnValue));
}
}
}
if (!writeBuffer.isEmpty()) {
//doOneRecord(writeBuffer, Constant.INSERTMODE);
this.doBatchRecordWithPartitionSort(writeBuffer, Constant.INSERTMODE, bufferSize, batchSize);
writeBuffer.clear();
}
// 2个缓冲最多一个不为空同时
if (null != deleteBuffer && !deleteBuffer.isEmpty()) {
//doOneRecord(deleteBuffer, Constant.DELETEMODE);
this.doBatchRecordWithPartitionSort(deleteBuffer, Constant.DELETEMODE, bufferSize, batchSize);
deleteBuffer.clear();
}
} catch (Exception e) {
throw DataXException.asDataXException(
DBUtilErrorCode.WRITE_DATA_ERROR, e);
} finally {
writeBuffer.clear();
DBUtil.closeDBResources(null, null, connection);
}
}
/**
* @param bufferSize datax缓冲记录条数
* @param batchSize datax向ads系统一次发送数据条数
* @param buffer datax缓冲区
* @param mode 实时表模式insert 或者 stream
*/
private void doBatchRecordWithPartitionSort(List<Record> buffer, String mode, int bufferSize, int batchSize) throws SQLException {
//warn: 排序会影响数据插入顺序, 如果源头没有数据约束, 排序可能造成数据不一致, 快速排序是一种不稳定的排序算法
//warn: 不明确配置bufferSize或者小于batchSize的情况下，不要进行排序;如果缓冲区实际内容条数少于batchSize也不排序了，最后一次的余量
int recordBufferedNumber = buffer.size();
if (bufferSize > batchSize && recordBufferedNumber > batchSize && this.partitionColumnIndex >= 0) {
final int partitionColumnIndex = this.partitionColumnIndex;
final int partitionCount = this.partitionCount;
Collections.sort(buffer, new Comparator<Record>() {
@Override
public int compare(Record record1, Record record2) {
int hashPartition1 = AdsInsertProxy.getHashPartition(record1.getColumn(partitionColumnIndex).asString(), partitionCount);
int hashPartition2 = AdsInsertProxy.getHashPartition(record2.getColumn(partitionColumnIndex).asString(), partitionCount);
return hashPartition1 - hashPartition2;
}
});
}
// 将缓冲区的Record输出到ads, 使用recordBufferedNumber哦
for (int i = 0; i < recordBufferedNumber; i += batchSize) {
int toIndex = i + batchSize;
if (toIndex > recordBufferedNumber) {
toIndex = recordBufferedNumber;
}
this.doBatchRecord(buffer.subList(i, toIndex), mode);
}
}
private void doBatchRecord(final List<Record> buffer, final String mode) throws SQLException {
List<Class<?>> retryExceptionClasss = new ArrayList<Class<?>>();
retryExceptionClasss.add(com.mysql.jdbc.exceptions.jdbc4.CommunicationsException.class);
retryExceptionClasss.add(java.net.SocketException.class);
try {
RetryUtil.executeWithRetry(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
doBatchRecordDml(buffer, mode);
return true;
}
}, this.retryTimeUpperLimit, 2000L, true, retryExceptionClasss);
} catch (SQLException e) {
LOG.warn(String.format("after retry %s times, doBatchRecord meet a exception: ", this.retryTimeUpperLimit), e);
LOG.info("try to re execute for each record...");
doOneRecord(buffer, mode);
// below is the old way
// for (Record eachRecord : buffer) {
// this.taskPluginCollector.collectDirtyRecord(eachRecord, e);
// }
} catch (Exception e) {
throw DataXException.asDataXException(
DBUtilErrorCode.WRITE_DATA_ERROR, e);
}
}
//warn: ADS 无法支持事物roll back都是不管用
@SuppressWarnings("resource")
private void doBatchRecordDml(List<Record> buffer, String mode) throws Exception {
Statement statement = null;
String sql = null;
try {
int bufferSize = buffer.size();
if (buffer.isEmpty()) {
return;
}
StringBuilder sqlSb = new StringBuilder();
// connection.setAutoCommit(true);
//mysql impl warn: if a database access error occurs or this method is called on a closed connection throw SQLException
statement = this.currentConnection.createStatement();
sqlSb.append(this.generateDmlSql(this.currentConnection, buffer.get(0), mode));
for (int i = 1; i < bufferSize; i++) {
Record record = buffer.get(i);
this.appendDmlSqlValues(this.currentConnection, record, sqlSb, mode);
}
sql = sqlSb.toString();
if (IS_DEBUG_ENABLE) {
LOG.debug(sql);
}
@SuppressWarnings("unused")
int status = statement.executeUpdate(sql);
sql = null;
} catch (SQLException e) {
LOG.warn("doBatchRecordDml meet a exception: " + sql, e);
Exception eachException = e;
int maxIter = 0;// 避免死循环
while (null != eachException && maxIter < AdsInsertProxy.MAX_EXCEPTION_CAUSE_ITER) {
if (this.isRetryable(eachException)) {
LOG.warn("doBatchRecordDml meet a retry exception: " + e.getMessage());
this.currentConnection = AdsUtil.getAdsConnect(this.configuration);
throw eachException;
} else {
try {
Throwable causeThrowable = eachException.getCause();
eachException = causeThrowable == null ? null : (Exception) causeThrowable;
} catch (Exception castException) {
LOG.warn("doBatchRecordDml meet a no! retry exception: " + e.getMessage());
throw e;
}
}
maxIter++;
}
throw e;
} catch (Exception e) {
LOG.error("插入异常, sql: " + sql);
throw DataXException.asDataXException(
DBUtilErrorCode.WRITE_DATA_ERROR, e);
} finally {
DBUtil.closeDBResources(statement, null);
}
}
private void doOneRecord(List<Record> buffer, final String mode) {
List<Class<?>> retryExceptionClasss = new ArrayList<Class<?>>();
retryExceptionClasss.add(com.mysql.jdbc.exceptions.jdbc4.CommunicationsException.class);
retryExceptionClasss.add(java.net.SocketException.class);
for (final Record record : buffer) {
try {
RetryUtil.executeWithRetry(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
doOneRecordDml(record, mode);
return true;
}
}, this.retryTimeUpperLimit, 2000L, true, retryExceptionClasss);
} catch (Exception e) {
// 不能重试的一行，记录脏数据
this.taskPluginCollector.collectDirtyRecord(record, e);
}
}
}
@SuppressWarnings("resource")
private void doOneRecordDml(Record record, String mode) throws Exception {
Statement statement = null;
String sql = null;
try {
// connection.setAutoCommit(true);
statement = this.currentConnection.createStatement();
sql = generateDmlSql(this.currentConnection, record, mode);
if (IS_DEBUG_ENABLE) {
LOG.debug(sql);
}
@SuppressWarnings("unused")
int status = statement.executeUpdate(sql);
sql = null;
} catch (SQLException e) {
LOG.error("doOneDml meet a exception: " + sql, e);
//need retry before record dirty data
//this.taskPluginCollector.collectDirtyRecord(record, e);
// 更新当前可用连接
Exception eachException = e;
int maxIter = 0;// 避免死循环
while (null != eachException && maxIter < AdsInsertProxy.MAX_EXCEPTION_CAUSE_ITER) {
if (this.isRetryable(eachException)) {
LOG.warn("doOneDml meet a retry exception: " + e.getMessage());
this.currentConnection = AdsUtil.getAdsConnect(this.configuration);
throw eachException;
} else {
try {
Throwable causeThrowable = eachException.getCause();
eachException = causeThrowable == null ? null : (Exception) causeThrowable;
} catch (Exception castException) {
LOG.warn("doOneDml meet a no! retry exception: " + e.getMessage());
throw e;
}
}
maxIter++;
}
throw e;
} catch (Exception e) {
LOG.error("插入异常, sql: " + sql);
throw DataXException.asDataXException(
DBUtilErrorCode.WRITE_DATA_ERROR, e);
} finally {
DBUtil.closeDBResources(statement, null);
}
}
private boolean isRetryable(Throwable e) {
Class<?> meetExceptionClass = e.getClass();
if (meetExceptionClass == com.mysql.jdbc.exceptions.jdbc4.CommunicationsException.class) {
return true;
}
if (meetExceptionClass == java.net.SocketException.class) {
return true;
}
return false;
}
private String generateDmlSql(Connection connection, Record record, String mode) throws SQLException {
String sql = null;
StringBuilder sqlSb = new StringBuilder();
if (mode.equalsIgnoreCase(Constant.INSERTMODE)) {
sqlSb.append(this.insertSqlPrefix);
sqlSb.append("(");
int columnsSize = this.columns.size();
for (int i = 0; i < columnsSize; i++) {
if ((i + 1) != columnsSize) {
sqlSb.append("?,");
} else {
sqlSb.append("?");
}
}
sqlSb.append(")");
//mysql impl warn: if a database access error occurs or this method is called on a closed connection
PreparedStatement statement = connection.prepareStatement(sqlSb.toString());
for (int i = 0; i < this.columns.size(); i++) {
int preparedParamsIndex = i;
if (Constant.STREAMMODE.equalsIgnoreCase(this.writeMode)) {
if (preparedParamsIndex >= this.opColumnIndex) {
preparedParamsIndex = i + 1;
}
}
String columnName = this.columns.get(i);
int columnSqltype = this.userConfigColumnsMetaData.get(columnName).getLeft();
prepareColumnTypeValue(statement, columnSqltype, record.getColumn(preparedParamsIndex), i, columnName);
}
sql = ((JDBC4PreparedStatement) statement).asSql();
DBUtil.closeDBResources(statement, null);
} else {
sqlSb.append(this.deleteSqlPrefix);
sqlSb.append("(");
Set<Entry<String, Integer>> primaryEntrySet = this.primaryKeyNameIndexMap.entrySet();
int entrySetSize = primaryEntrySet.size();
int i = 0;
for (Entry<String, Integer> eachEntry : primaryEntrySet) {
if ((i + 1) != entrySetSize) {
sqlSb.append(String.format(" (%s = ?) and ", eachEntry.getKey()));
} else {
sqlSb.append(String.format(" (%s = ?) ", eachEntry.getKey()));
}
i++;
}
sqlSb.append(")");
//mysql impl warn: if a database access error occurs or this method is called on a closed connection
PreparedStatement statement = connection.prepareStatement(sqlSb.toString());
i = 0;
//ads的real time表只能是1级分区、且分区列类型是long, 但是这里是需要主键删除的
for (Entry<String, Integer> each : primaryEntrySet) {
String columnName = each.getKey();
int columnSqlType = this.userConfigColumnsMetaData.get(columnName).getLeft();
int primaryKeyInUserConfigIndex = this.primaryKeyNameIndexMap.get(columnName);
if (primaryKeyInUserConfigIndex >= this.opColumnIndex) {
primaryKeyInUserConfigIndex++;
}
prepareColumnTypeValue(statement, columnSqlType, record.getColumn(primaryKeyInUserConfigIndex), i, columnName);
i++;
}
sql = ((JDBC4PreparedStatement) statement).asSql();
DBUtil.closeDBResources(statement, null);
}
return sql;
}
private void appendDmlSqlValues(Connection connection, Record record, StringBuilder sqlSb, String mode) throws SQLException {
String sqlResult = this.generateDmlSql(connection, record, mode);
if (mode.equalsIgnoreCase(Constant.INSERTMODE)) {
sqlSb.append(",");
sqlSb.append(sqlResult.substring(this.insertSqlPrefix.length()));
} else {
// 之前已经充分增加过括号了
sqlSb.append(" or ");
sqlSb.append(sqlResult.substring(this.deleteSqlPrefix.length()));
}
}
private void prepareColumnTypeValue(PreparedStatement statement, int columnSqltype, Column column, int preparedPatamIndex, String columnName) throws SQLException {
java.util.Date utilDate;
switch (columnSqltype) {
case Types.CHAR:
case Types.NCHAR:
case Types.CLOB:
case Types.NCLOB:
case Types.VARCHAR:
case Types.LONGVARCHAR:
case Types.NVARCHAR:
case Types.LONGNVARCHAR:
String strValue = column.asString();
statement.setString(preparedPatamIndex + 1, strValue);
break;
case Types.SMALLINT:
case Types.INTEGER:
case Types.BIGINT:
case Types.NUMERIC:
case Types.DECIMAL:
case Types.REAL:
String numValue = column.asString();
if (emptyAsNull && "".equals(numValue) || numValue == null) {
//statement.setObject(preparedPatamIndex + 1, null);
statement.setNull(preparedPatamIndex + 1, Types.BIGINT);
} else {
statement.setLong(preparedPatamIndex + 1, column.asLong());
}
break;
case Types.FLOAT:
case Types.DOUBLE:
String floatValue = column.asString();
if (emptyAsNull && "".equals(floatValue) || floatValue == null) {
//statement.setObject(preparedPatamIndex + 1, null);
statement.setNull(preparedPatamIndex + 1, Types.DOUBLE);
} else {
statement.setDouble(preparedPatamIndex + 1, column.asDouble());
}
break;
//tinyint is a little special in some database like mysql {boolean->tinyint(1)}
case Types.TINYINT:
Long longValue = column.asLong();
if (null == longValue) {
statement.setNull(preparedPatamIndex + 1, Types.BIGINT);
} else {
statement.setLong(preparedPatamIndex + 1, longValue);
}
break;
case Types.DATE:
java.sql.Date sqlDate = null;
try {
if ("".equals(column.getRawData())) {
utilDate = null;
} else {
utilDate = column.asDate();
}
} catch (DataXException e) {
throw new SQLException(String.format(
"Date 类型转换错误：[%s]", column));
}
if (null != utilDate) {
sqlDate = new java.sql.Date(utilDate.getTime());
}
statement.setDate(preparedPatamIndex + 1, sqlDate);
break;
case Types.TIME:
java.sql.Time sqlTime = null;
try {
if ("".equals(column.getRawData())) {
utilDate = null;
} else {
utilDate = column.asDate();
}
} catch (DataXException e) {
throw new SQLException(String.format(
"TIME 类型转换错误：[%s]", column));
}
if (null != utilDate) {
sqlTime = new java.sql.Time(utilDate.getTime());
}
statement.setTime(preparedPatamIndex + 1, sqlTime);
break;
case Types.TIMESTAMP:
java.sql.Timestamp sqlTimestamp = null;
try {
if ("".equals(column.getRawData())) {
utilDate = null;
} else {
utilDate = column.asDate();
}
} catch (DataXException e) {
throw new SQLException(String.format(
"TIMESTAMP 类型转换错误：[%s]", column));
}
if (null != utilDate) {
sqlTimestamp = new java.sql.Timestamp(
utilDate.getTime());
}
statement.setTimestamp(preparedPatamIndex + 1, sqlTimestamp);
break;
case Types.BOOLEAN:
//case Types.BIT: ads 没有bit
Boolean booleanValue = column.asBoolean();
if (null == booleanValue) {
statement.setNull(preparedPatamIndex + 1, Types.BOOLEAN);
} else {
statement.setBoolean(preparedPatamIndex + 1, booleanValue);
}
break;
default:
Pair<Integer, String> columnMetaPair = this.userConfigColumnsMetaData.get(columnName);
throw DataXException
.asDataXException(
DBUtilErrorCode.UNSUPPORTED_TYPE,
String.format(
"您的配置文件中的列配置信息有误. 因为DataX 不支持数据库写入这种字段类型. 字段名:[%s], 字段类型:[%s], 字段Java类型:[%s]. 请修改表中该字段的类型或者不同步该字段.",
columnName, columnMetaPair.getRight(), columnMetaPair.getLeft()));
}
}
private static int getHashPartition(String value, int totalHashPartitionNum) {
long crc32 = (value == null ? getCRC32("-1") : getCRC32(value));
return (int) (crc32 % totalHashPartitionNum);
}
private static long getCRC32(String value) {
Checksum checksum = new CRC32();
byte[] bytes = value.getBytes();
checksum.update(bytes, 0, bytes.length);
return checksum.getValue();
}
@Override
public void closeResource() {
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.insert;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.common.util.ListUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.AdsException;
import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
import com.alibaba.datax.plugin.writer.adswriter.load.AdsHelper;
import com.alibaba.datax.plugin.writer.adswriter.util.AdsUtil;
import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
import com.alibaba.datax.plugin.writer.adswriter.util.Key;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class AdsInsertUtil {
private static final Logger LOG = LoggerFactory
.getLogger(AdsInsertUtil.class);
public static TableInfo getAdsTableInfo(Configuration conf) {
AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
TableInfo tableInfo= null;
try {
tableInfo = adsHelper.getTableInfo(conf.getString(Key.ADS_TABLE));
} catch (AdsException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
}
return tableInfo;
}
/*
* 返回列顺序为ads建表列顺序
* */
public static List<String> getAdsTableColumnNames(Configuration conf) {
List<String> tableColumns = new ArrayList<String>();
AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
TableInfo tableInfo= null;
String adsTable = conf.getString(Key.ADS_TABLE);
try {
tableInfo = adsHelper.getTableInfo(adsTable);
} catch (AdsException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
}
List<ColumnInfo> columnInfos = tableInfo.getColumns();
for(ColumnInfo columnInfo: columnInfos) {
tableColumns.add(columnInfo.getName());
}
LOG.info("table:[{}] all columns:[\n{}\n].", adsTable, StringUtils.join(tableColumns, ","));
return tableColumns;
}
public static Map<String, Pair<Integer,String>> getColumnMetaData
(Configuration configuration, List<String> userColumns) {
Map<String, Pair<Integer,String>> columnMetaData = new HashMap<String, Pair<Integer,String>>();
List<ColumnInfo> columnInfoList = getAdsTableColumns(configuration);
for(String column : userColumns) {
if (column.startsWith(Constant.ADS_QUOTE_CHARACTER) && column.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
column = column.substring(1, column.length() - 1);
}
for (ColumnInfo columnInfo : columnInfoList) {
if(column.equalsIgnoreCase(columnInfo.getName())) {
Pair<Integer,String> eachPair = new ImmutablePair<Integer, String>(columnInfo.getDataType().sqlType, columnInfo.getDataType().name);
columnMetaData.put(columnInfo.getName(), eachPair);
}
}
}
return columnMetaData;
}
public static Map<String, Pair<Integer,String>> getColumnMetaData(TableInfo tableInfo, List<String> userColumns){
Map<String, Pair<Integer,String>> columnMetaData = new HashMap<String, Pair<Integer,String>>();
List<ColumnInfo> columnInfoList = tableInfo.getColumns();
for(String column : userColumns) {
if (column.startsWith(Constant.ADS_QUOTE_CHARACTER) && column.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
column = column.substring(1, column.length() - 1);
}
for (ColumnInfo columnInfo : columnInfoList) {
if(column.equalsIgnoreCase(columnInfo.getName())) {
Pair<Integer,String> eachPair = new ImmutablePair<Integer, String>(columnInfo.getDataType().sqlType, columnInfo.getDataType().name);
columnMetaData.put(columnInfo.getName(), eachPair);
}
}
}
return columnMetaData;
}
/*
* 返回列顺序为ads建表列顺序
* */
public static List<ColumnInfo> getAdsTableColumns(Configuration conf) {
AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
TableInfo tableInfo= null;
String adsTable = conf.getString(Key.ADS_TABLE);
try {
tableInfo = adsHelper.getTableInfo(adsTable);
} catch (AdsException e) {
throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
}
List<ColumnInfo> columnInfos = tableInfo.getColumns();
return columnInfos;
}
public static void dealColumnConf(Configuration originalConfig, List<String> tableColumns) {
List<String> userConfiguredColumns = originalConfig.getList(Key.COLUMN, String.class);
if (null == userConfiguredColumns || userConfiguredColumns.isEmpty()) {
throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE,
"您的配置文件中的列配置信息有误. 因为您未配置写入数据库表的列名称，DataX获取不到列信息. 请检查您的配置并作出修改.");
} else {
if (1 == userConfiguredColumns.size() && "*".equals(userConfiguredColumns.get(0))) {
LOG.warn("您的配置文件中的列配置信息存在风险. 因为您配置的写入数据库表的列为*，当您的表字段个数、类型有变动时，可能影响任务正确性甚至会运行出错。请检查您的配置并作出修改.");
// 回填其值，需要以 String 的方式转交后续处理
originalConfig.set(Key.COLUMN, tableColumns);
} else if (userConfiguredColumns.size() > tableColumns.size()) {
throw DataXException.asDataXException(DBUtilErrorCode.ILLEGAL_VALUE,
String.format("您的配置文件中的列配置信息有误. 因为您所配置的写入数据库表的字段个数:%s 大于目的表的总字段总个数:%s. 请检查您的配置并作出修改.",
userConfiguredColumns.size(), tableColumns.size()));
} else {
// 确保用户配置的 column 不重复
ListUtil.makeSureNoValueDuplicate(userConfiguredColumns, false);
// 检查列是否都为数据库表中正确的列（通过执行一次 select column from table 进行判断）
// ListUtil.makeSureBInA(tableColumns, userConfiguredColumns, true);
// 支持关键字和保留字, ads列是不区分大小写的
List<String> removeQuotedColumns = new ArrayList<String>();
for (String each : userConfiguredColumns) {
if (each.startsWith(Constant.ADS_QUOTE_CHARACTER) && each.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
removeQuotedColumns.add(each.substring(1, each.length() - 1));
} else {
removeQuotedColumns.add(each);
}
}
ListUtil.makeSureBInA(tableColumns, removeQuotedColumns, false);
}
}
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsProxy.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.insert;
import com.alibaba.datax.common.plugin.RecordReceiver;
import java.sql.Connection;
public interface AdsProxy {
public abstract void startWriteWithConnection(RecordReceiver recordReceiver, Connection connection,
int columnNumber);
public void closeResource();
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/OperationType.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.insert;
public enum OperationType {
// i: insert uo:before image uu:before image un: after image d: delete
// u:update
I("i"), UO("uo"), UU("uu"), UN("un"), D("d"), U("u"), UNKNOWN("unknown"), ;
private OperationType(String type) {
this.type = type;
}
private String type;
public String getType() {
return this.type;
}
public static OperationType asOperationType(String type) {
if ("i".equalsIgnoreCase(type)) {
return I;
} else if ("uo".equalsIgnoreCase(type)) {
return UO;
} else if ("uu".equalsIgnoreCase(type)) {
return UU;
} else if ("un".equalsIgnoreCase(type)) {
return UN;
} else if ("d".equalsIgnoreCase(type)) {
return D;
} else if ("u".equalsIgnoreCase(type)) {
return U;
} else {
return UNKNOWN;
}
}
public boolean isInsertTemplate() {
switch (this) {
// 建议merge 过后应该只有I和U两类
case I:
case UO:
case UU:
case UN:
case U:
return true;
case D:
return false;
default:
return false;
}
}
public boolean isDeleteTemplate() {
switch (this) {
// 建议merge 过后应该只有I和U两类
case I:
case UO:
case UU:
case UN:
case U:
return false;
case D:
return true;
default:
return false;
}
}
public boolean isLegal() {
return this.type != UNKNOWN.getType();
}
@Override
public String toString() {
return this.name();
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java
================================================
/**
*
*/
package com.alibaba.datax.plugin.writer.adswriter.load;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.util.RetryUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtil;
import com.alibaba.datax.plugin.writer.adswriter.AdsException;
import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnDataType;
import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
import com.alibaba.datax.plugin.writer.adswriter.util.AdsUtil;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.Callable;
public class AdsHelper {
private static final Logger LOG = LoggerFactory
.getLogger(AdsHelper.class);
private String adsURL;
private String userName;
private String password;
private String schema;
private Long socketTimeout;
private String suffix;
public AdsHelper(String adsUrl, String userName, String password, String schema, Long socketTimeout, String suffix) {
this.adsURL = adsUrl;
this.userName = userName;
this.password = password;
this.schema = schema;
this.socketTimeout = socketTimeout;
this.suffix = suffix;
}
public String getAdsURL() {
return adsURL;
}
public void setAdsURL(String adsURL) {
this.adsURL = adsURL;
}
public String getUserName() {
return userName;
}
public void setUserName(String userName) {
this.userName = userName;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
/**
* Obtain the table meta information.
*
* @param table The table
* @return The table meta information
* @throws com.alibaba.datax.plugin.writer.adswriter.AdsException
*/
public TableInfo getTableInfo(String table) throws AdsException {
if (table == null) {
throw new AdsException(AdsException.ADS_TABLEMETA_TABLE_NULL, "Table is null.", null);
}
if (adsURL == null) {
throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
}
if (userName == null) {
throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
"ADS JDBC connection user name was not set.", null);
}
if (password == null) {
throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
null);
}
if (schema == null) {
throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
null);
}
Connection connection = null;
Statement statement = null;
ResultSet rs = null;
try {
Class.forName("com.mysql.jdbc.Driver");
String url = AdsUtil.prepareJdbcUrl(this.adsURL, this.schema, this.socketTimeout, this.suffix);
Properties connectionProps = new Properties();
connectionProps.put("user", userName);
connectionProps.put("password", password);
connection = DriverManager.getConnection(url, connectionProps);
statement = connection.createStatement();
// ads 表名、schema名不区分大小写, 提高用户易用性, 注意列顺序性
String columnMetaSql = String.format("select ordinal_position,column_name,data_type,type_name,column_comment from information_schema.columns where table_schema = `'%s'` and table_name = `'%s'` order by ordinal_position", schema.toLowerCase(), table.toLowerCase());
LOG.info(String.format("检查列信息sql语句:%s", columnMetaSql));
rs = statement.executeQuery(columnMetaSql);
TableInfo tableInfo = new TableInfo();
List<ColumnInfo> columnInfoList = new ArrayList<ColumnInfo>();
while (DBUtil.asyncResultSetNext(rs)) {
ColumnInfo columnInfo = new ColumnInfo();
columnInfo.setOrdinal(rs.getInt(1));
columnInfo.setName(rs.getString(2));
//columnInfo.setDataType(ColumnDataType.getDataType(rs.getInt(3))); //for ads version < 0.7
//columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(3).toUpperCase())); //for ads version 0.8
columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(4).toUpperCase())); //for ads version 0.8 & 0.7
columnInfo.setComment(rs.getString(5));
columnInfoList.add(columnInfo);
}
if (columnInfoList.isEmpty()) {
throw DataXException.asDataXException(AdsWriterErrorCode.NO_ADS_TABLE, table + "不存在或者查询不到列信息. ");
}
tableInfo.setColumns(columnInfoList);
tableInfo.setTableSchema(schema);
tableInfo.setTableName(table);
DBUtil.closeDBResources(rs, statement, null);
String tableMetaSql = String.format("select update_type, partition_type, partition_column, partition_count, primary_key_columns from information_schema.tables where table_schema = `'%s'` and table_name = `'%s'`", schema.toLowerCase(), table.toLowerCase());
LOG.info(String.format("检查表信息sql语句:%s", tableMetaSql));
statement = connection.createStatement();
rs = statement.executeQuery(tableMetaSql);
while (DBUtil.asyncResultSetNext(rs)) {
tableInfo.setUpdateType(rs.getString(1));
tableInfo.setPartitionType(rs.getString(2));
tableInfo.setPartitionColumn(rs.getString(3));
tableInfo.setPartitionCount(rs.getInt(4));
//primary_key_columns ads主键是逗号分隔的，可以有多个
String primaryKeyColumns = rs.getString(5);
if (StringUtils.isNotBlank(primaryKeyColumns)) {
tableInfo.setPrimaryKeyColumns(Arrays.asList(StringUtils.split(primaryKeyColumns, ",")));
} else {
tableInfo.setPrimaryKeyColumns(null);
}
break;
}
DBUtil.closeDBResources(rs, statement, null);
return tableInfo;
} catch (ClassNotFoundException e) {
throw new AdsException(AdsException.OTHER, e.getMessage(), e);
} catch (SQLException e) {
throw new AdsException(AdsException.OTHER, e.getMessage(), e);
} catch ( DataXException e) {
throw e;
} catch (Exception e) {
throw new AdsException(AdsException.OTHER, e.getMessage(), e);
} finally {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
// Ignore exception
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException e) {
// Ignore exception
}
}
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
// Ignore exception
}
}
}
}
/**
* Submit LOAD DATA command.
*
* @param table The target ADS table
* @param partition The partition option in the form of "(partition_name,...)"
* @param sourcePath The source path
* @param overwrite
* @return
* @throws AdsException
*/
public String loadData(String table, String partition, String sourcePath, boolean overwrite) throws AdsException {
if (table == null) {
throw new AdsException(AdsException.ADS_LOADDATA_TABLE_NULL, "ADS LOAD DATA table is null.", null);
}
if (sourcePath == null) {
throw new AdsException(AdsException.ADS_LOADDATA_SOURCEPATH_NULL, "ADS LOAD DATA source path is null.",
null);
}
if (adsURL == null) {
throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
}
if (userName == null) {
throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
"ADS JDBC connection user name was not set.", null);
}
if (password == null) {
throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
null);
}
if (schema == null) {
throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
null);
}
StringBuilder sb = new StringBuilder();
sb.append("LOAD DATA FROM ");
if (sourcePath.startsWith("'") && sourcePath.endsWith("'")) {
sb.append(sourcePath);
} else {
sb.append("'" + sourcePath + "'");
}
if (overwrite) {
sb.append(" OVERWRITE");
}
sb.append(" INTO TABLE ");
sb.append(schema + "." + table);
if (partition != null && !partition.trim().equals("")) {
String partitionTrim = partition.trim();
if(partitionTrim.startsWith("(") && partitionTrim.endsWith(")")) {
sb.append(" PARTITION " + partition);
} else {
sb.append(" PARTITION " + "(" + partition + ")");
}
}
Connection connection = null;
Statement statement = null;
ResultSet rs = null;
try {
Class.forName("com.mysql.jdbc.Driver");
String url = AdsUtil.prepareJdbcUrl(this.adsURL, this.schema, this.socketTimeout, this.suffix);
Properties connectionProps = new Properties();
connectionProps.put("user", userName);
connectionProps.put("password", password);
connection = DriverManager.getConnection(url, connectionProps);
statement = connection.createStatement();
LOG.info("正在从ODPS数据库导数据到ADS中: "+sb.toString());
LOG.info("由于ADS的限制，ADS导数据最少需要20分钟，请耐心等待");
rs = statement.executeQuery(sb.toString());
String jobId = null;
while (DBUtil.asyncResultSetNext(rs)) {
jobId = rs.getString(1);
}
if (jobId == null) {
throw new AdsException(AdsException.ADS_LOADDATA_JOBID_NOT_AVAIL,
"Job id is not available for the submitted LOAD DATA." + jobId, null);
}
return jobId;
} catch (ClassNotFoundException e) {
throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
} catch (SQLException e) {
throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
} catch (Exception e) {
throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
} finally {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
// Ignore exception
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException e) {
// Ignore exception
}
}
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
// Ignore exception
}
}
}
}
/**
* Check the load data job status.
*
* @param jobId The job id to
* @return true if load data job succeeded, false if load data job failed.
* @throws AdsException
*/
public boolean checkLoadDataJobStatus(String jobId) throws AdsException {
if (adsURL == null) {
throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
}
if (userName == null) {
throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET,
"ADS JDBC connection user name was not set.", null);
}
if (password == null) {
throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.",
null);
}
if (schema == null) {
throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.",
null);
}
try {
String state = this.checkLoadDataJobStatusWithRetry(jobId);
if (state == null) {
throw new AdsException(AdsException.JOB_NOT_EXIST, "Target job does not exist for id: " + jobId, null);
}
if (state.equals("SUCCEEDED")) {
return true;
} else if (state.equals("FAILED")) {
throw new AdsException(AdsException.JOB_FAILED, "Target job failed for id: " + jobId, null);
} else {
return false;
}
} catch (Exception e) {
throw new AdsException(AdsException.OTHER, e.getMessage(), e);
}
}
private String checkLoadDataJobStatusWithRetry(final String jobId)
throws AdsException {
try {
Class.forName("com.mysql.jdbc.Driver");
final String finalAdsUrl = this.adsURL;
final String finalSchema = this.schema;
final Long finalSocketTimeout = this.socketTimeout;
final String suffix = this.suffix;
return RetryUtil.executeWithRetry(new Callable<String>() {
@Override
public String call() throws Exception {
Connection connection = null;
Statement statement = null;
ResultSet rs = null;
try {
String url = AdsUtil.prepareJdbcUrl(finalAdsUrl, finalSchema, finalSocketTimeout, suffix);
Properties connectionProps = new Properties();
connectionProps.put("user", userName);
connectionProps.put("password", password);
connection = DriverManager.getConnection(url,
connectionProps);
statement = connection.createStatement();
String sql = "select state from information_schema.job_instances where job_id like '"
+ jobId + "'";
rs = statement.executeQuery(sql);
String state = null;
while (DBUtil.asyncResultSetNext(rs)) {
state = rs.getString(1);
}
return state;
} finally {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
// Ignore exception
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException e) {
// Ignore exception
}
}
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
// Ignore exception
}
}
}
}
}, 3, 1000L, true);
} catch (Exception e) {
throw new AdsException(AdsException.OTHER, e.getMessage(), e);
}
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.load;
import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnDataType;
import com.alibaba.datax.plugin.writer.adswriter.ads.ColumnInfo;
import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
import com.alibaba.datax.plugin.writer.adswriter.odps.DataType;
import com.alibaba.datax.plugin.writer.adswriter.odps.FieldSchema;
import com.alibaba.datax.plugin.writer.adswriter.odps.TableMeta;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
/**
* Table meta helper for ADS writer.
*
* @since 0.0.1
*/
public class TableMetaHelper {
private TableMetaHelper() {
}
/**
* Create temporary ODPS table.
*
* @param tableMeta table meta
* @param lifeCycle for temporary table
* @return ODPS temporary table meta
*/
public static TableMeta createTempODPSTable(TableInfo tableMeta, int lifeCycle) {
TableMeta tempTable = new TableMeta();
tempTable.setComment(tableMeta.getComments());
tempTable.setLifeCycle(lifeCycle);
String tableSchema = tableMeta.getTableSchema();
String tableName = tableMeta.getTableName();
tempTable.setTableName(generateTempTableName(tableSchema, tableName));
List<FieldSchema> tempColumns = new ArrayList<FieldSchema>();
List<ColumnInfo> columns = tableMeta.getColumns();
for (ColumnInfo column : columns) {
FieldSchema tempColumn = new FieldSchema();
tempColumn.setName(column.getName());
tempColumn.setType(toODPSDataType(column.getDataType()));
tempColumn.setComment(column.getComment());
tempColumns.add(tempColumn);
}
tempTable.setCols(tempColumns);
tempTable.setPartitionKeys(null);
return tempTable;
}
private static String toODPSDataType(ColumnDataType columnDataType) {
int type;
switch (columnDataType.type) {
case ColumnDataType.BOOLEAN:
type = DataType.STRING;
break;
case ColumnDataType.BYTE:
case ColumnDataType.SHORT:
case ColumnDataType.INT:
case ColumnDataType.LONG:
type = DataType.INTEGER;
break;
case ColumnDataType.DECIMAL:
case ColumnDataType.DOUBLE:
case ColumnDataType.FLOAT:
type = DataType.DOUBLE;
break;
case ColumnDataType.DATE:
case ColumnDataType.TIME:
case ColumnDataType.TIMESTAMP:
case ColumnDataType.STRING:
case ColumnDataType.MULTI_VALUE:
type = DataType.STRING;
break;
default:
throw new IllegalArgumentException("columnDataType=" + columnDataType);
}
return DataType.toString(type);
}
private static String generateTempTableName(String tableSchema, String tableName) {
int randNum = 1000 + new Random(System.currentTimeMillis()).nextInt(1000);
return tableSchema + "__" + tableName + "_" + System.currentTimeMillis() + randNum;
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.load;
import com.alibaba.datax.common.util.Configuration;
/**
* Created by xiafei.qiuxf on 15/4/13.
*/
public class TransferProjectConf {
public final static String KEY_ACCESS_ID = "odps.accessId";
public final static String KEY_ACCESS_KEY = "odps.accessKey";
public final static String KEY_ACCOUNT = "odps.account";
public final static String KEY_ODPS_SERVER = "odps.odpsServer";
public final static String KEY_ODPS_TUNNEL = "odps.tunnelServer";
public final static String KEY_PROJECT = "odps.project";
private String accessId;
private String accessKey;
private String account;
private String odpsServer;
private String odpsTunnel;
private String project;
public static TransferProjectConf create(Configuration adsWriterConf) {
TransferProjectConf res = new TransferProjectConf();
res.accessId = adsWriterConf.getString(KEY_ACCESS_ID);
res.accessKey = adsWriterConf.getString(KEY_ACCESS_KEY);
res.account = adsWriterConf.getString(KEY_ACCOUNT);
res.odpsServer = adsWriterConf.getString(KEY_ODPS_SERVER);
res.odpsTunnel = adsWriterConf.getString(KEY_ODPS_TUNNEL);
res.project = adsWriterConf.getString(KEY_PROJECT);
return res;
}
public String getAccessId() {
return accessId;
}
public String getAccessKey() {
return accessKey;
}
public String getAccount() {
return account;
}
public String getOdpsServer() {
return odpsServer;
}
public String getOdpsTunnel() {
return odpsTunnel;
}
public String getProject() {
return project;
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.odps;
/**
* ODPS 数据类型.
* <p>
* 当前定义了如下类型：
* <ul>
* <li>INTEGER
* <li>DOUBLE
* <li>BOOLEAN
* <li>STRING
* <li>DATETIME
* </ul>
* </p>
*
* @since 0.0.1
*/
public class DataType {
public final static byte INTEGER = 0;
public final static byte DOUBLE = 1;
public final static byte BOOLEAN = 2;
public final static byte STRING = 3;
public final static byte DATETIME = 4;
public static String toString(int type) {
gitextract_n5eqdejp/ ├── .gitignore ├── NOTICE ├── README.md ├── adbmysqlwriter/ │ ├── doc/ │ │ └── adbmysqlwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── adbmysqlwriter/ │ │ └── AdbMysqlWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── adbpgwriter/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── doc/ │ │ └── adbpgwriter.md │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── adbpgwriter/ │ │ ├── AdbpgWriter.java │ │ ├── copy/ │ │ │ ├── Adb4pgClientProxy.java │ │ │ └── AdbProxy.java │ │ ├── package-info.java │ │ └── util/ │ │ ├── Adb4pgUtil.java │ │ ├── Constant.java │ │ └── Key.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── adswriter/ │ ├── doc/ │ │ └── adswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── adswriter/ │ │ ├── AdsException.java │ │ ├── AdsWriter.java │ │ ├── AdsWriterErrorCode.java │ │ ├── ads/ │ │ │ ├── ColumnDataType.java │ │ │ ├── ColumnInfo.java │ │ │ ├── TableInfo.java │ │ │ └── package-info.java │ │ ├── insert/ │ │ │ ├── AdsClientProxy.java │ │ │ ├── AdsInsertProxy.java │ │ │ ├── AdsInsertUtil.java │ │ │ ├── AdsProxy.java │ │ │ └── OperationType.java │ │ ├── load/ │ │ │ ├── AdsHelper.java │ │ │ ├── TableMetaHelper.java │ │ │ └── TransferProjectConf.java │ │ ├── odps/ │ │ │ ├── DataType.java │ │ │ ├── FieldSchema.java │ │ │ ├── TableMeta.java │ │ │ └── package-info.java │ │ ├── package-info.java │ │ └── util/ │ │ ├── AdsUtil.java │ │ ├── Constant.java │ │ └── Key.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── cassandrareader/ │ ├── doc/ │ │ └── cassandrareader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── cassandrareader/ │ │ ├── CassandraReader.java │ │ ├── CassandraReaderErrorCode.java │ │ ├── CassandraReaderHelper.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ └── LocalStrings_zh_TW.properties │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── cassandrawriter/ │ ├── doc/ │ │ └── cassandrawriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── cassandrawriter/ │ │ ├── CassandraWriter.java │ │ ├── CassandraWriterErrorCode.java │ │ ├── CassandraWriterHelper.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ └── LocalStrings_zh_TW.properties │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── clickhousereader/ │ ├── doc/ │ │ └── clickhousereader.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ └── clickhousereader/ │ │ │ └── ClickhouseReader.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ └── resources/ │ ├── basic1.json │ └── basic1.sql ├── clickhousewriter/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── clickhousewriter/ │ │ ├── ClickhouseWriter.java │ │ └── ClickhouseWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── common/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── common/ │ ├── base/ │ │ └── BaseObject.java │ ├── constant/ │ │ ├── CommonConstant.java │ │ └── PluginType.java │ ├── element/ │ │ ├── BoolColumn.java │ │ ├── BytesColumn.java │ │ ├── Column.java │ │ ├── ColumnCast.java │ │ ├── DateColumn.java │ │ ├── DoubleColumn.java │ │ ├── LongColumn.java │ │ ├── OverFlowUtil.java │ │ ├── Record.java │ │ └── StringColumn.java │ ├── exception/ │ │ ├── CommonErrorCode.java │ │ ├── DataXException.java │ │ └── ExceptionTracker.java │ ├── plugin/ │ │ ├── AbstractJobPlugin.java │ │ ├── AbstractPlugin.java │ │ ├── AbstractTaskPlugin.java │ │ ├── JobPluginCollector.java │ │ ├── PluginCollector.java │ │ ├── Pluginable.java │ │ ├── RecordReceiver.java │ │ ├── RecordSender.java │ │ └── TaskPluginCollector.java │ ├── spi/ │ │ ├── ErrorCode.java │ │ ├── Hook.java │ │ ├── Reader.java │ │ └── Writer.java │ ├── statistics/ │ │ ├── PerfRecord.java │ │ ├── PerfTrace.java │ │ └── VMInfo.java │ └── util/ │ ├── Configuration.java │ ├── ConfigurationUtil.java │ ├── DESCipher.java │ ├── DataXCaseEnvUtil.java │ ├── FilterUtil.java │ ├── HostUtils.java │ ├── LimitLogger.java │ ├── ListUtil.java │ ├── LocalStrings.properties │ ├── LocalStrings_en_US.properties │ ├── LocalStrings_ja_JP.properties │ ├── LocalStrings_zh_CN.properties │ ├── LocalStrings_zh_HK.properties │ ├── LocalStrings_zh_TW.properties │ ├── LoggerFunction.java │ ├── MessageSource.java │ ├── RangeSplitUtil.java │ ├── RetryUtil.java │ └── StrUtil.java ├── core/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── conf/ │ │ ├── .secret.properties │ │ ├── core.json │ │ └── logback.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ ├── core/ │ │ │ ├── AbstractContainer.java │ │ │ ├── Engine.java │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ ├── container/ │ │ │ │ └── util/ │ │ │ │ ├── HookInvoker.java │ │ │ │ └── JobAssignUtil.java │ │ │ ├── job/ │ │ │ │ ├── JobContainer.java │ │ │ │ ├── meta/ │ │ │ │ │ ├── ExecuteMode.java │ │ │ │ │ └── State.java │ │ │ │ └── scheduler/ │ │ │ │ ├── AbstractScheduler.java │ │ │ │ └── processinner/ │ │ │ │ ├── ProcessInnerScheduler.java │ │ │ │ └── StandAloneScheduler.java │ │ │ ├── statistics/ │ │ │ │ ├── communication/ │ │ │ │ │ ├── Communication.java │ │ │ │ │ ├── CommunicationTool.java │ │ │ │ │ └── LocalTGCommunicationManager.java │ │ │ │ ├── container/ │ │ │ │ │ ├── collector/ │ │ │ │ │ │ ├── AbstractCollector.java │ │ │ │ │ │ └── ProcessInnerCollector.java │ │ │ │ │ ├── communicator/ │ │ │ │ │ │ ├── AbstractContainerCommunicator.java │ │ │ │ │ │ ├── job/ │ │ │ │ │ │ │ └── StandAloneJobContainerCommunicator.java │ │ │ │ │ │ └── taskgroup/ │ │ │ │ │ │ ├── AbstractTGContainerCommunicator.java │ │ │ │ │ │ └── StandaloneTGContainerCommunicator.java │ │ │ │ │ └── report/ │ │ │ │ │ ├── AbstractReporter.java │ │ │ │ │ └── ProcessInnerReporter.java │ │ │ │ └── plugin/ │ │ │ │ ├── DefaultJobPluginCollector.java │ │ │ │ └── task/ │ │ │ │ ├── AbstractTaskPluginCollector.java │ │ │ │ ├── HttpPluginCollector.java │ │ │ │ ├── StdoutPluginCollector.java │ │ │ │ └── util/ │ │ │ │ └── DirtyRecord.java │ │ │ ├── taskgroup/ │ │ │ │ ├── TaskGroupContainer.java │ │ │ │ ├── TaskMonitor.java │ │ │ │ └── runner/ │ │ │ │ ├── AbstractRunner.java │ │ │ │ ├── ReaderRunner.java │ │ │ │ ├── TaskGroupContainerRunner.java │ │ │ │ └── WriterRunner.java │ │ │ ├── transport/ │ │ │ │ ├── channel/ │ │ │ │ │ ├── Channel.java │ │ │ │ │ └── memory/ │ │ │ │ │ └── MemoryChannel.java │ │ │ │ ├── exchanger/ │ │ │ │ │ ├── BufferedRecordExchanger.java │ │ │ │ │ ├── BufferedRecordTransformerExchanger.java │ │ │ │ │ ├── RecordExchanger.java │ │ │ │ │ └── TransformerExchanger.java │ │ │ │ ├── record/ │ │ │ │ │ ├── DefaultRecord.java │ │ │ │ │ └── TerminateRecord.java │ │ │ │ └── transformer/ │ │ │ │ ├── ComplexTransformerProxy.java │ │ │ │ ├── DigestTransformer.java │ │ │ │ ├── FilterTransformer.java │ │ │ │ ├── GroovyTransformer.java │ │ │ │ ├── GroovyTransformerStaticUtil.java │ │ │ │ ├── PadTransformer.java │ │ │ │ ├── ReplaceTransformer.java │ │ │ │ ├── SubstrTransformer.java │ │ │ │ ├── TransformerErrorCode.java │ │ │ │ ├── TransformerExecution.java │ │ │ │ ├── TransformerExecutionParas.java │ │ │ │ ├── TransformerInfo.java │ │ │ │ └── TransformerRegistry.java │ │ │ └── util/ │ │ │ ├── ClassSize.java │ │ │ ├── ClassUtil.java │ │ │ ├── ConfigParser.java │ │ │ ├── ConfigurationValidate.java │ │ │ ├── ErrorRecordChecker.java │ │ │ ├── ExceptionTracker.java │ │ │ ├── FrameworkErrorCode.java │ │ │ ├── HttpClientUtil.java │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ ├── SecretUtil.java │ │ │ ├── TransformerUtil.java │ │ │ └── container/ │ │ │ ├── ClassLoaderSwapper.java │ │ │ ├── CoreConstant.java │ │ │ ├── JarLoader.java │ │ │ └── LoadUtil.java │ │ └── dataxservice/ │ │ └── face/ │ │ └── domain/ │ │ └── enums/ │ │ ├── EnumStrVal.java │ │ ├── EnumVal.java │ │ ├── ExecuteMode.java │ │ └── State.java │ ├── job/ │ │ └── job.json │ └── script/ │ └── Readme.md ├── databendwriter/ │ ├── doc/ │ │ ├── databendwriter-CN.md │ │ └── databendwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── databendwriter/ │ │ ├── DatabendWriter.java │ │ ├── DatabendWriterErrorCode.java │ │ └── util/ │ │ └── DatabendWriterUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── datahubreader/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── datahubreader/ │ │ ├── Constant.java │ │ ├── DatahubClientHelper.java │ │ ├── DatahubReader.java │ │ ├── DatahubReaderErrorCode.java │ │ ├── DatahubReaderUtils.java │ │ ├── DatahubWriterErrorCode.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ └── LocalStrings_zh_TW.properties │ └── resources/ │ ├── job_config_template.json │ └── plugin.json ├── datahubwriter/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── datahubwriter/ │ │ ├── DatahubClientHelper.java │ │ ├── DatahubWriter.java │ │ ├── DatahubWriterErrorCode.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ └── LocalStrings_zh_TW.properties │ └── resources/ │ ├── job_config_template.json │ └── plugin.json ├── datax-example/ │ ├── datax-example-core/ │ │ ├── pom.xml │ │ └── src/ │ │ ├── main/ │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── alibaba/ │ │ │ │ └── datax/ │ │ │ │ └── example/ │ │ │ │ ├── ExampleContainer.java │ │ │ │ ├── Main.java │ │ │ │ └── util/ │ │ │ │ ├── ExampleConfigParser.java │ │ │ │ └── PathUtil.java │ │ │ └── resources/ │ │ │ └── example/ │ │ │ └── conf/ │ │ │ └── core.json │ │ └── test/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── example/ │ │ │ └── util/ │ │ │ └── PathUtilTest.java │ │ └── resources/ │ │ └── pathTest.json │ ├── datax-example-neo4j/ │ │ ├── pom.xml │ │ └── src/ │ │ └── test/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── example/ │ │ │ └── neo4j/ │ │ │ └── StreamReader2Neo4jWriterTest.java │ │ └── resources/ │ │ └── streamreader2neo4j.json │ ├── datax-example-streamreader/ │ │ ├── pom.xml │ │ └── src/ │ │ └── test/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── example/ │ │ │ └── streamreader/ │ │ │ └── StreamReader2StreamWriterTest.java │ │ └── resources/ │ │ └── stream2stream.json │ ├── doc/ │ │ └── README.md │ └── pom.xml ├── dataxPluginDev.md ├── dorisreader/ │ ├── doc/ │ │ └── dorisreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── dorisreader/ │ │ ├── DorisReader.java │ │ └── DorisReaderErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── doriswriter/ │ ├── doc/ │ │ ├── doriswriter.md │ │ └── mysql2doris.json │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── doriswriter/ │ │ ├── DelimiterParser.java │ │ ├── DorisBaseCodec.java │ │ ├── DorisCodec.java │ │ ├── DorisCodecFactory.java │ │ ├── DorisCsvCodec.java │ │ ├── DorisJsonCodec.java │ │ ├── DorisStreamLoadObserver.java │ │ ├── DorisUtil.java │ │ ├── DorisWriter.java │ │ ├── DorisWriterExcetion.java │ │ ├── DorisWriterManager.java │ │ ├── Keys.java │ │ └── WriterTuple.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── drdsreader/ │ ├── doc/ │ │ └── drdsreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── drdsreader/ │ │ ├── DrdsReader.java │ │ ├── DrdsReaderErrorCode.java │ │ └── DrdsReaderSplitUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── drdswriter/ │ ├── doc/ │ │ └── drdswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── drdswriter/ │ │ └── DrdsWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── elasticsearchwriter/ │ ├── README.md │ ├── build.sh │ ├── doc/ │ │ └── elasticsearchwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── elasticsearchwriter/ │ │ ├── ElasticSearchClient.java │ │ ├── ElasticSearchColumn.java │ │ ├── ElasticSearchFieldType.java │ │ ├── ElasticSearchWriter.java │ │ ├── ElasticSearchWriterErrorCode.java │ │ ├── JsonPathUtil.java │ │ ├── JsonUtil.java │ │ ├── Key.java │ │ ├── NoReRunException.java │ │ ├── PartitionColumn.java │ │ ├── PrimaryKeyInfo.java │ │ └── jest/ │ │ ├── ClusterInfo.java │ │ ├── ClusterInfoResult.java │ │ └── PutMapping7.java │ └── resources/ │ └── plugin.json ├── ftpreader/ │ ├── doc/ │ │ └── ftpreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── ftpreader/ │ │ ├── Constant.java │ │ ├── FtpHelper.java │ │ ├── FtpReader.java │ │ ├── FtpReaderErrorCode.java │ │ ├── Key.java │ │ ├── SftpHelper.java │ │ └── StandardFtpHelper.java │ └── resources/ │ ├── plugin-template.json │ ├── plugin.json │ └── plugin_job_template.json ├── ftpwriter/ │ ├── doc/ │ │ ├── .gitkeep │ │ └── ftpwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── ftpwriter/ │ │ ├── FtpWriter.java │ │ ├── FtpWriterErrorCode.java │ │ ├── Key.java │ │ └── util/ │ │ ├── Constant.java │ │ ├── IFtpHelper.java │ │ ├── SftpHelperImpl.java │ │ └── StandardFtpHelperImpl.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── gaussdbreader/ │ ├── doc/ │ │ └── gaussdbreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── gaussdbreader/ │ │ ├── Constant.java │ │ └── GaussDbReader.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── gaussdbwriter/ │ ├── doc/ │ │ └── gaussdbwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── gaussdbwriter/ │ │ └── GaussDbWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── gdbreader/ │ ├── doc/ │ │ └── gdbreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── gdbreader/ │ │ ├── GdbReader.java │ │ ├── GdbReaderErrorCode.java │ │ ├── Key.java │ │ ├── mapping/ │ │ │ ├── DefaultGdbMapper.java │ │ │ ├── MappingRule.java │ │ │ ├── MappingRuleFactory.java │ │ │ └── ValueType.java │ │ ├── model/ │ │ │ ├── AbstractGdbGraph.java │ │ │ ├── GdbElement.java │ │ │ ├── GdbGraph.java │ │ │ └── ScriptGdbGraph.java │ │ └── util/ │ │ └── ConfigHelper.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── gdbwriter/ │ ├── doc/ │ │ └── gdbwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── gdbwriter/ │ │ ├── GdbWriter.java │ │ ├── GdbWriterErrorCode.java │ │ ├── Key.java │ │ ├── client/ │ │ │ ├── GdbGraphManager.java │ │ │ └── GdbWriterConfig.java │ │ ├── mapping/ │ │ │ ├── DefaultGdbMapper.java │ │ │ ├── GdbMapper.java │ │ │ ├── MapperConfig.java │ │ │ ├── MappingRule.java │ │ │ ├── MappingRuleFactory.java │ │ │ └── ValueType.java │ │ ├── model/ │ │ │ ├── AbstractGdbGraph.java │ │ │ ├── GdbEdge.java │ │ │ ├── GdbElement.java │ │ │ ├── GdbGraph.java │ │ │ ├── GdbVertex.java │ │ │ └── ScriptGdbGraph.java │ │ └── util/ │ │ ├── ConfigHelper.java │ │ └── GdbDuplicateIdException.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hbase094xreader/ │ ├── doc/ │ │ ├── .gitkeep │ │ └── hbase094xreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── hbase094xreader/ │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase094xHelper.java │ │ ├── Hbase094xReader.java │ │ ├── Hbase094xReaderErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── HbaseColumnCell.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── MultiVersionFixedColumnTask.java │ │ ├── MultiVersionTask.java │ │ └── NormalTask.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hbase094xwriter/ │ ├── doc/ │ │ ├── .gitkeep │ │ └── hbase094xwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── hbase094xwriter/ │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase094xHelper.java │ │ ├── Hbase094xWriter.java │ │ ├── Hbase094xWriterErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── NormalTask.java │ │ └── NullModeType.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hbase11xreader/ │ ├── doc/ │ │ ├── .gitkeep │ │ └── hbase11xreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── hbase11xreader/ │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase11xHelper.java │ │ ├── Hbase11xReader.java │ │ ├── Hbase11xReaderErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── HbaseColumnCell.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── MultiVersionDynamicColumnTask.java │ │ ├── MultiVersionFixedColumnTask.java │ │ ├── MultiVersionTask.java │ │ └── NormalTask.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hbase11xsqlreader/ │ ├── doc/ │ │ └── hbase11xsqlreader.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ └── hbase11xsqlreader/ │ │ │ ├── HadoopSerializationUtil.java │ │ │ ├── HbaseSQLHelper.java │ │ │ ├── HbaseSQLReader.java │ │ │ ├── HbaseSQLReaderConfig.java │ │ │ ├── HbaseSQLReaderErrorCode.java │ │ │ ├── HbaseSQLReaderTask.java │ │ │ ├── Key.java │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ └── LocalStrings_zh_CN.properties │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── reader/ │ └── hbase11xsqlreader/ │ ├── HbaseSQLHelperTest.java │ └── HbaseSQLReaderTaskTest.java ├── hbase11xsqlwriter/ │ ├── doc/ │ │ └── hbase11xsqlwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── hbase11xsqlwriter/ │ │ ├── Constant.java │ │ ├── HbaseSQLHelper.java │ │ ├── HbaseSQLWriter.java │ │ ├── HbaseSQLWriterConfig.java │ │ ├── HbaseSQLWriterErrorCode.java │ │ ├── HbaseSQLWriterTask.java │ │ ├── Key.java │ │ ├── NullModeType.java │ │ └── ThinClientPTable.java │ └── resources/ │ └── plugin.json ├── hbase11xwriter/ │ ├── doc/ │ │ ├── .gitkeep │ │ └── hbase11xwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── hbase11xwriter/ │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase11xHelper.java │ │ ├── Hbase11xWriter.java │ │ ├── Hbase11xWriterErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── MultiVersionTask.java │ │ ├── NormalTask.java │ │ └── NullModeType.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hbase20xsqlreader/ │ ├── doc/ │ │ └── hbase20xsqlreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── hbase20xsqlreader/ │ │ ├── Constant.java │ │ ├── HBase20SQLReaderHelper.java │ │ ├── HBase20xSQLReader.java │ │ ├── HBase20xSQLReaderErrorCode.java │ │ ├── HBase20xSQLReaderTask.java │ │ └── Key.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hbase20xsqlwriter/ │ ├── doc/ │ │ └── hbase20xsqlwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── hbase20xsqlwriter/ │ │ ├── Constant.java │ │ ├── HBase20xSQLHelper.java │ │ ├── HBase20xSQLWriter.java │ │ ├── HBase20xSQLWriterErrorCode.java │ │ ├── HBase20xSQLWriterTask.java │ │ ├── Key.java │ │ └── NullModeType.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hdfsreader/ │ ├── doc/ │ │ └── hdfsreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── hdfsreader/ │ │ ├── Constant.java │ │ ├── DFSUtil.java │ │ ├── HdfsFileType.java │ │ ├── HdfsPathFilter.java │ │ ├── HdfsReader.java │ │ ├── HdfsReaderErrorCode.java │ │ ├── Key.java │ │ ├── ParquetMessageHelper.java │ │ └── ParquetMeta.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hdfswriter/ │ ├── doc/ │ │ └── hdfswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── hdfswriter/ │ │ ├── Constant.java │ │ ├── HdfsHelper.java │ │ ├── HdfsWriter.java │ │ ├── HdfsWriterErrorCode.java │ │ ├── Key.java │ │ ├── ParquetFileProccessor.java │ │ ├── ParquetFileSupport.java │ │ └── SupportHiveDataType.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── hologresjdbcwriter/ │ ├── doc/ │ │ └── hologresjdbcwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── hologresjdbcwriter/ │ │ ├── BaseWriter.java │ │ ├── Constant.java │ │ ├── HologresJdbcWriter.java │ │ ├── Key.java │ │ └── util/ │ │ ├── ConfLoader.java │ │ ├── OriginalConfPretreatmentUtil.java │ │ └── WriterUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── introduction.md ├── kingbaseesreader/ │ ├── doc/ │ │ └── kingbaseesreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── kingbaseesreader/ │ │ ├── Constant.java │ │ └── KingbaseesReader.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── kingbaseeswriter/ │ ├── doc/ │ │ └── kingbaseeswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── kingbaseeswriter/ │ │ └── KingbaseesWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── kuduwriter/ │ ├── README.md │ ├── doc/ │ │ └── kuduwirter.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── q1/ │ │ │ ├── datax/ │ │ │ │ └── plugin/ │ │ │ │ └── writer/ │ │ │ │ └── kudu11xwriter/ │ │ │ │ ├── ColumnType.java │ │ │ │ ├── Constant.java │ │ │ │ ├── InsertModeType.java │ │ │ │ ├── Key.java │ │ │ │ ├── Kudu11xHelper.java │ │ │ │ ├── Kudu11xWriter.java │ │ │ │ ├── Kudu11xWriterErrorcode.java │ │ │ │ └── KuduWriterTask.java │ │ │ └── kudu/ │ │ │ └── conf/ │ │ │ └── KuduConfig.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ └── java/ │ └── com/ │ └── dai/ │ └── test.java ├── license.txt ├── loghubreader/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── loghubreader/ │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── LogHubReader.java │ │ └── LogHubReaderErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── loghubwriter/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── loghubwriter/ │ │ ├── Key.java │ │ ├── LogHubWriter.java │ │ └── LogHubWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── milvuswriter/ │ ├── doc/ │ │ └── milvuswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── milvuswriter/ │ │ ├── KeyConstant.java │ │ ├── MilvusBufferWriter.java │ │ ├── MilvusClient.java │ │ ├── MilvusColumn.java │ │ ├── MilvusCreateCollection.java │ │ ├── MilvusWriter.java │ │ ├── MilvusWriterErrorCode.java │ │ └── enums/ │ │ ├── SchemaCreateModeEnum.java │ │ └── WriteModeEnum.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── mongodbreader/ │ ├── doc/ │ │ └── mongodbreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── mongodbreader/ │ │ ├── KeyConstant.java │ │ ├── MongoDBReader.java │ │ ├── MongoDBReaderErrorCode.java │ │ └── util/ │ │ ├── CollectionSplitUtil.java │ │ └── MongoUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── mongodbwriter/ │ ├── doc/ │ │ └── mongodbwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── mongodbwriter/ │ │ ├── KeyConstant.java │ │ ├── MongoDBWriter.java │ │ ├── MongoDBWriterErrorCode.java │ │ └── util/ │ │ └── MongoUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── mysqlreader/ │ ├── doc/ │ │ └── mysqlreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── mysqlreader/ │ │ ├── MysqlReader.java │ │ └── MysqlReaderErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── mysqlwriter/ │ ├── doc/ │ │ └── mysqlwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── mysqlwriter/ │ │ └── MysqlWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── neo4jwriter/ │ ├── doc/ │ │ └── neo4jwriter.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── writer/ │ │ │ └── neo4jwriter/ │ │ │ ├── Neo4jClient.java │ │ │ ├── Neo4jWriter.java │ │ │ ├── adapter/ │ │ │ │ ├── DateAdapter.java │ │ │ │ └── ValueAdapter.java │ │ │ ├── config/ │ │ │ │ ├── ConfigConstants.java │ │ │ │ ├── Neo4jProperty.java │ │ │ │ └── Option.java │ │ │ ├── element/ │ │ │ │ └── PropertyType.java │ │ │ └── exception/ │ │ │ └── Neo4jErrorCode.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ ├── Neo4jWriterTest.java │ │ └── mock/ │ │ ├── MockRecord.java │ │ └── MockUtil.java │ └── resources/ │ ├── allTypeFieldNode.json │ ├── dynamicLabel.json │ ├── relationship.json │ └── streamreader2neo4j.json ├── obhbasereader/ │ ├── doc/ │ │ └── obhbasereader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── obhbasereader/ │ │ ├── Constant.java │ │ ├── HTableManager.java │ │ ├── HbaseColumnCell.java │ │ ├── HbaseReaderErrorCode.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ ├── LocalStrings_zh_TW.properties │ │ ├── ObHbaseReader.java │ │ ├── enums/ │ │ │ ├── ColumnType.java │ │ │ ├── FetchVersion.java │ │ │ └── ModeType.java │ │ ├── ext/ │ │ │ └── ServerConnectInfo.java │ │ ├── task/ │ │ │ ├── AbstractHbaseTask.java │ │ │ ├── AbstractScanReader.java │ │ │ ├── SQLNormalModeReader.java │ │ │ ├── ScanMultiVersionReader.java │ │ │ └── ScanNormalModeReader.java │ │ └── util/ │ │ ├── HbaseSplitUtil.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ ├── LocalStrings_zh_TW.properties │ │ ├── ObHbaseReaderUtil.java │ │ └── SqlReaderSplitUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── obhbasewriter/ │ ├── doc/ │ │ └── obhbasewriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── obhbasewriter/ │ │ ├── ColumnType.java │ │ ├── Config.java │ │ ├── ConfigKey.java │ │ ├── ConfigValidator.java │ │ ├── Constant.java │ │ ├── Hbase094xWriterErrorCode.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ ├── LocalStrings_zh_TW.properties │ │ ├── ModeType.java │ │ ├── NullModeType.java │ │ ├── ObHTableInfo.java │ │ ├── ObHbaseWriter.java │ │ ├── ext/ │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ ├── ObDataSourceErrorCode.java │ │ │ ├── ObHbaseTableHolder.java │ │ │ └── ServerConnectInfo.java │ │ ├── task/ │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ ├── MultiVersionWriteTask.java │ │ │ ├── NormalWriteTask.java │ │ │ ├── ObHBaseWriteTask.java │ │ │ └── PutTask.java │ │ └── util/ │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ ├── LocalStrings_zh_TW.properties │ │ └── ObHbaseWriterUtils.java │ └── resources/ │ └── plugin.json ├── oceanbasev10reader/ │ ├── doc/ │ │ └── oceanbasev10reader.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ └── oceanbasev10reader/ │ │ │ ├── Config.java │ │ │ ├── OceanBaseReader.java │ │ │ ├── ext/ │ │ │ │ ├── Constant.java │ │ │ │ ├── ObReaderKey.java │ │ │ │ ├── ReaderJob.java │ │ │ │ └── ReaderTask.java │ │ │ └── util/ │ │ │ ├── ExecutorTemplate.java │ │ │ ├── ObReaderUtils.java │ │ │ ├── PartInfo.java │ │ │ ├── PartType.java │ │ │ ├── PartitionSplitUtil.java │ │ │ └── TaskContext.java │ │ └── resources/ │ │ └── plugin.json │ └── test/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── reader/ │ └── oceanbasev10reader/ │ └── util/ │ └── ObReaderUtilsTest.java ├── oceanbasev10writer/ │ ├── doc/ │ │ └── oceanbasev10writer.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── oceanbasev10writer/ │ │ ├── Config.java │ │ ├── OceanBaseV10Writer.java │ │ ├── common/ │ │ │ ├── Table.java │ │ │ └── TableCache.java │ │ ├── directPath/ │ │ │ ├── AbstractRestrictedConnection.java │ │ │ ├── AbstractRestrictedPreparedStatement.java │ │ │ ├── DirectLoaderBuilder.java │ │ │ ├── DirectPathConnection.java │ │ │ ├── DirectPathConstants.java │ │ │ ├── DirectPathPreparedStatement.java │ │ │ └── ObTableDirectLoad.java │ │ ├── ext/ │ │ │ ├── AbstractConnHolder.java │ │ │ ├── ConnHolder.java │ │ │ ├── DataBaseWriterBuffer.java │ │ │ ├── DirectPathAbstractConnHolder.java │ │ │ ├── DirectPathConnHolder.java │ │ │ ├── OBDataSourceV10.java │ │ │ ├── OCJConnHolder.java │ │ │ ├── ObClientConnHolder.java │ │ │ ├── ObDataSourceErrorCode.java │ │ │ └── ServerConnectInfo.java │ │ ├── part/ │ │ │ ├── IObPartCalculator.java │ │ │ ├── ObPartitionCalculatorV1.java │ │ │ └── ObPartitionCalculatorV2.java │ │ ├── task/ │ │ │ ├── AbstractInsertTask.java │ │ │ ├── ColumnMetaCache.java │ │ │ ├── ConcurrentTableWriterTask.java │ │ │ ├── DirectPathInsertTask.java │ │ │ ├── InsertTask.java │ │ │ ├── SingleTableWriterTask.java │ │ │ └── WriterThreadPool.java │ │ └── util/ │ │ ├── DbUtils.java │ │ └── ObWriterUtils.java │ └── resources/ │ └── plugin.json ├── ocswriter/ │ ├── doc/ │ │ └── ocswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── ocswriter/ │ │ ├── Key.java │ │ ├── OcsWriter.java │ │ └── utils/ │ │ ├── CommonUtils.java │ │ ├── ConfigurationChecker.java │ │ └── OcsWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── odpsreader/ │ ├── doc/ │ │ └── odpsreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── odpsreader/ │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── InternalColumnInfo.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── OdpsReader.java │ │ ├── OdpsReaderErrorCode.java │ │ ├── ReaderProxy.java │ │ └── util/ │ │ ├── LocalStrings.properties │ │ ├── OdpsExceptionMsg.java │ │ ├── OdpsSplitUtil.java │ │ ├── OdpsUtil.java │ │ ├── SqliteUtil.java │ │ └── UserConfiguredPartitionClassification.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── odpswriter/ │ ├── doc/ │ │ └── odpswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── odpswriter/ │ │ ├── Constant.java │ │ ├── DateTransForm.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── OdpsWriter.java │ │ ├── OdpsWriterErrorCode.java │ │ ├── OdpsWriterProxy.java │ │ ├── model/ │ │ │ ├── PartitionInfo.java │ │ │ ├── UserDefinedFunction.java │ │ │ └── UserDefinedFunctionRule.java │ │ └── util/ │ │ ├── CustomPartitionUtils.java │ │ ├── LocalStrings.properties │ │ ├── OdpsExceptionMsg.java │ │ └── OdpsUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── opentsdbreader/ │ ├── doc/ │ │ └── opentsdbreader.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ ├── conn/ │ │ │ │ ├── CliQuery.java │ │ │ │ ├── Connection4TSDB.java │ │ │ │ ├── DataPoint4TSDB.java │ │ │ │ ├── DumpSeries.java │ │ │ │ ├── OpenTSDBConnection.java │ │ │ │ └── OpenTSDBDump.java │ │ │ ├── opentsdbreader/ │ │ │ │ ├── Constant.java │ │ │ │ ├── Key.java │ │ │ │ ├── OpenTSDBReader.java │ │ │ │ └── OpenTSDBReaderErrorCode.java │ │ │ └── util/ │ │ │ ├── HttpUtils.java │ │ │ ├── TSDBUtils.java │ │ │ └── TimeUtils.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── reader/ │ ├── conn/ │ │ └── OpenTSDBConnectionTest.java │ └── util/ │ ├── Const.java │ ├── HttpUtilsTest.java │ ├── TSDBTest.java │ └── TimeUtilsTest.java ├── oraclereader/ │ ├── doc/ │ │ └── oraclereader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── oraclereader/ │ │ ├── Constant.java │ │ ├── OracleReader.java │ │ └── OracleReaderErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── oraclewriter/ │ ├── doc/ │ │ └── oraclewriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── oraclewriter/ │ │ ├── OracleWriter.java │ │ └── OracleWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── oscarwriter/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── oscarwriter/ │ │ ├── OscarWriter.java │ │ └── OscarWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── ossreader/ │ ├── doc/ │ │ └── ossreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── ossreader/ │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── OssInputStream.java │ │ ├── OssReader.java │ │ ├── OssReaderErrorCode.java │ │ └── util/ │ │ ├── HdfsParquetUtil.java │ │ ├── OssSplitUtil.java │ │ └── OssUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── osswriter/ │ ├── doc/ │ │ └── osswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── osswriter/ │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── OssSingleObject.java │ │ ├── OssWriter.java │ │ ├── OssWriterErrorCode.java │ │ ├── OssWriterProxy.java │ │ ├── parquet/ │ │ │ ├── ParquetFileProccessor.java │ │ │ └── ParquetFileSupport.java │ │ └── util/ │ │ ├── HandlerUtil.java │ │ ├── HdfsParquetUtil.java │ │ └── OssUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── otsreader/ │ ├── doc/ │ │ └── otsreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── otsreader/ │ │ ├── IOtsReaderMasterProxy.java │ │ ├── IOtsReaderSlaveProxy.java │ │ ├── OtsReader.java │ │ ├── OtsReaderMasterProxy.java │ │ ├── OtsReaderSlaveMetaProxy.java │ │ ├── OtsReaderSlaveMultiVersionProxy.java │ │ ├── OtsReaderSlaveNormalProxy.java │ │ ├── OtsReaderSlaveProxyOld.java │ │ ├── adaptor/ │ │ │ ├── ColumnAdaptor.java │ │ │ └── PrimaryKeyValueAdaptor.java │ │ ├── callable/ │ │ │ ├── GetFirstRowPrimaryKeyCallable.java │ │ │ ├── GetRangeCallable.java │ │ │ ├── GetRangeCallableOld.java │ │ │ ├── GetTableMetaCallable.java │ │ │ ├── GetTimeseriesSplitCallable.java │ │ │ └── ScanTimeseriesDataCallable.java │ │ ├── model/ │ │ │ ├── DefaultNoRetry.java │ │ │ ├── OTSColumn.java │ │ │ ├── OTSConf.java │ │ │ ├── OTSConst.java │ │ │ ├── OTSCriticalException.java │ │ │ ├── OTSErrorCode.java │ │ │ ├── OTSMode.java │ │ │ ├── OTSMultiVersionConf.java │ │ │ ├── OTSPrimaryKeyColumn.java │ │ │ └── OTSRange.java │ │ └── utils/ │ │ ├── Common.java │ │ ├── CommonOld.java │ │ ├── CompareHelper.java │ │ ├── Constant.java │ │ ├── DefaultNoRetry.java │ │ ├── GsonParser.java │ │ ├── Key.java │ │ ├── OtsHelper.java │ │ ├── OtsReaderError.java │ │ ├── ParamChecker.java │ │ ├── ParamCheckerOld.java │ │ ├── ParamParser.java │ │ ├── RangeSplit.java │ │ ├── ReaderModelParser.java │ │ ├── RetryHelper.java │ │ ├── RetryHelperOld.java │ │ └── TranformHelper.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── otsstreamreader/ │ ├── README.md │ ├── pom.xml │ ├── src/ │ │ └── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ └── otsstreamreader/ │ │ │ └── internal/ │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ ├── OTSReaderError.java │ │ │ ├── OTSStreamReader.java │ │ │ ├── OTSStreamReaderException.java │ │ │ ├── OTSStreamReaderMasterProxy.java │ │ │ ├── OTSStreamReaderSlaveProxy.java │ │ │ ├── config/ │ │ │ │ ├── LocalStrings.properties │ │ │ │ ├── LocalStrings_en_US.properties │ │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ │ ├── Mode.java │ │ │ │ ├── OTSRetryStrategyForStreamReader.java │ │ │ │ ├── OTSStreamReaderConfig.java │ │ │ │ ├── OTSStreamReaderConstants.java │ │ │ │ └── StatusTableConstants.java │ │ │ ├── core/ │ │ │ │ ├── CheckpointTimeTracker.java │ │ │ │ ├── IStreamRecordSender.java │ │ │ │ ├── LocalStrings.properties │ │ │ │ ├── LocalStrings_en_US.properties │ │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ │ ├── MultiVerModeRecordSender.java │ │ │ │ ├── OTSStreamReaderChecker.java │ │ │ │ ├── RecordProcessor.java │ │ │ │ ├── ShardStatusChecker.java │ │ │ │ └── SingleVerAndUpOnlyModeRecordSender.java │ │ │ ├── model/ │ │ │ │ ├── LocalStrings.properties │ │ │ │ ├── LocalStrings_en_US.properties │ │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ │ ├── OTSErrorCode.java │ │ │ │ ├── OTSStreamJobShard.java │ │ │ │ ├── ShardCheckpoint.java │ │ │ │ └── StreamJob.java │ │ │ └── utils/ │ │ │ ├── ColumnValueTransformHelper.java │ │ │ ├── GsonParser.java │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ ├── LocalStrings_zh_CN.properties │ │ │ ├── LocalStrings_zh_HK.properties │ │ │ ├── LocalStrings_zh_TW.properties │ │ │ ├── OTSHelper.java │ │ │ ├── OTSStreamJobShardUtil.java │ │ │ ├── ParamChecker.java │ │ │ └── TimeUtils.java │ │ └── resources/ │ │ ├── log4j2.xml │ │ └── plugin.json │ └── tools/ │ ├── config.json │ ├── tablestore_streamreader_console.py │ └── tabulate.py ├── otswriter/ │ ├── doc/ │ │ └── otswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── otswriter/ │ │ ├── IOtsWriterMasterProxy.java │ │ ├── IOtsWriterSlaveProxy.java │ │ ├── Key.java │ │ ├── OTSCriticalException.java │ │ ├── OTSErrorCode.java │ │ ├── OtsWriter.java │ │ ├── OtsWriterError.java │ │ ├── OtsWriterMasterProxy.java │ │ ├── OtsWriterSlaveProxyMultiversion.java │ │ ├── OtsWriterSlaveProxyNormal.java │ │ ├── OtsWriterSlaveProxyOld.java │ │ ├── callable/ │ │ │ ├── BatchWriteRowCallable.java │ │ │ ├── GetTableMetaCallable.java │ │ │ ├── GetTableMetaCallableOld.java │ │ │ ├── PutRowChangeCallable.java │ │ │ ├── PutTimeseriesDataCallable.java │ │ │ └── UpdateRowChangeCallable.java │ │ ├── model/ │ │ │ ├── OTSAttrColumn.java │ │ │ ├── OTSBatchWriteRowTaskManager.java │ │ │ ├── OTSBatchWriterRowTask.java │ │ │ ├── OTSBlockingExecutor.java │ │ │ ├── OTSConf.java │ │ │ ├── OTSConst.java │ │ │ ├── OTSErrorMessage.java │ │ │ ├── OTSLine.java │ │ │ ├── OTSMode.java │ │ │ ├── OTSOpType.java │ │ │ ├── OTSSendBuffer.java │ │ │ ├── OTSTaskManagerInterface.java │ │ │ ├── OTSTimeseriesRowTask.java │ │ │ ├── OTSTimeseriesRowTaskManager.java │ │ │ ├── RowDeleteChangeWithRecord.java │ │ │ ├── RowPutChangeWithRecord.java │ │ │ └── RowUpdateChangeWithRecord.java │ │ └── utils/ │ │ ├── CalculateHelper.java │ │ ├── CollectorUtil.java │ │ ├── ColumnConversion.java │ │ ├── ColumnConversionOld.java │ │ ├── Common.java │ │ ├── CommonOld.java │ │ ├── DefaultNoRetry.java │ │ ├── GsonParser.java │ │ ├── LineAndError.java │ │ ├── ParamChecker.java │ │ ├── ParseRecord.java │ │ ├── RetryHelper.java │ │ ├── WithRecord.java │ │ ├── WriterModelParser.java │ │ └── WriterRetryPolicy.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── package.xml ├── plugin-rdbms-util/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── rdbms/ │ ├── reader/ │ │ ├── CommonRdbmsReader.java │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── ResultSetReadProxy.java │ │ └── util/ │ │ ├── HintUtil.java │ │ ├── ObVersion.java │ │ ├── OriginalConfPretreatmentUtil.java │ │ ├── PreCheckTask.java │ │ ├── ReaderSplitUtil.java │ │ └── SingleTableSplitUtil.java │ ├── util/ │ │ ├── ConnectionFactory.java │ │ ├── Constant.java │ │ ├── DBUtil.java │ │ ├── DBUtilErrorCode.java │ │ ├── DataBaseType.java │ │ ├── JdbcConnectionFactory.java │ │ ├── RdbmsException.java │ │ ├── RdbmsRangeSplitWrap.java │ │ ├── SplitedSlice.java │ │ └── TableExpandUtil.java │ └── writer/ │ ├── CommonRdbmsWriter.java │ ├── Constant.java │ ├── Key.java │ ├── MysqlWriterErrorCode.java │ └── util/ │ ├── OriginalConfPretreatmentUtil.java │ └── WriterUtil.java ├── plugin-unstructured-storage-util/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── unstructuredstorage/ │ ├── FileFormat.java │ ├── LocalStrings.properties │ ├── LocalStrings_en_US.properties │ ├── LocalStrings_ja_JP.properties │ ├── LocalStrings_zh_CN.properties │ ├── LocalStrings_zh_HK.properties │ ├── LocalStrings_zh_TW.properties │ ├── reader/ │ │ ├── ColumnEntry.java │ │ ├── Constant.java │ │ ├── ExpandLzopInputStream.java │ │ ├── Key.java │ │ ├── UnstructuredStorageReaderErrorCode.java │ │ ├── UnstructuredStorageReaderUtil.java │ │ ├── ZipCycleInputStream.java │ │ ├── binaryFileUtil/ │ │ │ ├── BinaryFileReaderUtil.java │ │ │ └── ByteUtils.java │ │ └── split/ │ │ ├── StartEndPair.java │ │ └── UnstructuredSplitUtil.java │ ├── util/ │ │ ├── ColumnTypeUtil.java │ │ └── HdfsUtil.java │ └── writer/ │ ├── Constant.java │ ├── DataXCsvWriter.java │ ├── Key.java │ ├── SqlWriter.java │ ├── TextCsvWriterManager.java │ ├── UnstructuredStorageWriterErrorCode.java │ ├── UnstructuredStorageWriterUtil.java │ ├── UnstructuredWriter.java │ └── binaryFileUtil/ │ ├── BinaryFileWriterErrorCode.java │ └── BinaryFileWriterUtil.java ├── pom.xml ├── postgresqlreader/ │ ├── doc/ │ │ └── postgresqlreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── postgresqlreader/ │ │ ├── Constant.java │ │ └── PostgresqlReader.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── postgresqlwriter/ │ ├── doc/ │ │ └── postgresqlwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── postgresqlwriter/ │ │ └── PostgresqlWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── rdbmsreader/ │ ├── doc/ │ │ └── rdbmsreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── rdbmsreader/ │ │ ├── Constant.java │ │ ├── RdbmsReader.java │ │ └── SubCommonRdbmsReader.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── rdbmswriter/ │ ├── doc/ │ │ └── rdbmswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── rdbmswriter/ │ │ ├── RdbmsWriter.java │ │ └── SubCommonRdbmsWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── rpm/ │ ├── t_dp_dw_datax_3_core_all-build.sh │ └── t_dp_dw_datax_3_hook_dqc-build.sh ├── selectdbwriter/ │ ├── doc/ │ │ ├── selectdbwriter.md │ │ └── stream2selectdb.json │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── selectdbwriter/ │ │ ├── BaseResponse.java │ │ ├── CopyIntoResp.java │ │ ├── CopySQLBuilder.java │ │ ├── DelimiterParser.java │ │ ├── HttpPostBuilder.java │ │ ├── HttpPutBuilder.java │ │ ├── Keys.java │ │ ├── SelectdbBaseCodec.java │ │ ├── SelectdbCodec.java │ │ ├── SelectdbCodecFactory.java │ │ ├── SelectdbCopyIntoObserver.java │ │ ├── SelectdbCsvCodec.java │ │ ├── SelectdbJsonCodec.java │ │ ├── SelectdbUtil.java │ │ ├── SelectdbWriter.java │ │ ├── SelectdbWriterException.java │ │ ├── SelectdbWriterManager.java │ │ └── WriterTuple.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── sqlserverreader/ │ ├── doc/ │ │ └── sqlserverreader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── sqlserverreader/ │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── SqlServerReader.java │ │ └── SqlServerReaderErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── sqlserverwriter/ │ ├── doc/ │ │ └── sqlserverwriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── sqlserverwriter/ │ │ ├── SqlServerWriter.java │ │ └── SqlServerWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── starrocksreader/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── starrocksreader/ │ │ └── StarRocksReader.java │ └── resources/ │ └── plugin.json ├── starrockswriter/ │ ├── doc/ │ │ └── starrockswriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── starrocks/ │ │ └── connector/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── starrockswriter/ │ │ ├── StarRocksWriter.java │ │ ├── StarRocksWriterOptions.java │ │ ├── manager/ │ │ │ ├── StarRocksFlushTuple.java │ │ │ ├── StarRocksStreamLoadFailedException.java │ │ │ ├── StarRocksStreamLoadVisitor.java │ │ │ └── StarRocksWriterManager.java │ │ ├── row/ │ │ │ ├── StarRocksBaseSerializer.java │ │ │ ├── StarRocksCsvSerializer.java │ │ │ ├── StarRocksDelimiterParser.java │ │ │ ├── StarRocksISerializer.java │ │ │ ├── StarRocksJsonSerializer.java │ │ │ └── StarRocksSerializerFactory.java │ │ └── util/ │ │ └── StarRocksWriterUtil.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── streamreader/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── streamreader/ │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── StreamReader.java │ │ └── StreamReaderErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── streamwriter/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── streamwriter/ │ │ ├── Key.java │ │ ├── StreamWriter.java │ │ └── StreamWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── sybasereader/ │ ├── doc/ │ │ └── sybasereader.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ └── sybasereader/ │ │ │ ├── Constants.java │ │ │ └── SybaseReader.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── reader/ │ └── sybasereader/ │ └── SybaseDatabaseUnitTest.java ├── sybasewriter/ │ ├── doc/ │ │ └── sybasewriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ └── java/ │ ├── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── sybasewriter/ │ │ └── SybaseWriter.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── tdenginereader/ │ ├── doc/ │ │ └── tdenginereader-CN.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ ├── TDengineReader.java │ │ │ └── TDengineReaderErrorCode.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ ├── TDengine2DMTest.java │ │ ├── TDengine2StreamTest.java │ │ └── TDengineReaderTest.java │ └── resources/ │ ├── t2dm.json │ ├── t2stream-1.json │ └── t2stream-2.json ├── tdenginewriter/ │ ├── doc/ │ │ ├── tdenginewriter-CN.md │ │ └── tdenginewriter.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── writer/ │ │ │ └── tdenginewriter/ │ │ │ ├── ColumnMeta.java │ │ │ ├── Constants.java │ │ │ ├── DataHandler.java │ │ │ ├── DefaultDataHandler.java │ │ │ ├── Key.java │ │ │ ├── OpentsdbDataHandler.java │ │ │ ├── SchemaManager.java │ │ │ ├── TDengineWriter.java │ │ │ ├── TDengineWriterErrorCode.java │ │ │ ├── TableMeta.java │ │ │ ├── TableType.java │ │ │ └── TimestampPrecision.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── tdenginewriter/ │ │ ├── Csv2TDengineTest.java │ │ ├── DM2TDengineTest.java │ │ ├── DefaultDataHandlerTest.java │ │ ├── Mongo2TDengineTest.java │ │ ├── Mysql2TDengineTest.java │ │ ├── Opentsdb2TDengineTest.java │ │ ├── SchemaManagerTest.java │ │ ├── Stream2TDengineTest.java │ │ ├── TDengine2TDengineTest.java │ │ └── TDengineWriterTest.java │ └── resources/ │ ├── csv2t.json │ ├── defaultJob.json │ ├── dm-schema.sql │ ├── dm2t-1.json │ ├── dm2t-2.json │ ├── dm2t-3.json │ ├── dm2t-4.json │ ├── incremental_sync/ │ │ ├── clean_env.sh │ │ ├── csv2t-jni.json │ │ ├── csv2t-restful.json │ │ ├── dm2t-jni.json │ │ ├── dm2t-restful.json │ │ ├── dm2t-update.json │ │ ├── dm2t_sync.sh │ │ ├── t2dm-jni.json │ │ ├── t2dm-restful.json │ │ └── upload.sh │ ├── m2t-1.json │ ├── mongo2t.json │ ├── o2t-1.json │ ├── t2t-1.json │ ├── t2t-2.json │ ├── t2t-3.json │ ├── t2t-4.json │ └── weather.csv ├── transformer/ │ ├── doc/ │ │ ├── .gitkeep │ │ └── transformer.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── transformer/ │ ├── ComplexTransformer.java │ └── Transformer.java ├── tsdbreader/ │ ├── doc/ │ │ └── tsdbreader.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── reader/ │ │ │ └── tsdbreader/ │ │ │ ├── Constant.java │ │ │ ├── Key.java │ │ │ ├── TSDBReader.java │ │ │ ├── TSDBReaderErrorCode.java │ │ │ ├── conn/ │ │ │ │ ├── Connection4TSDB.java │ │ │ │ ├── DataPoint4MultiFieldsTSDB.java │ │ │ │ ├── DataPoint4TSDB.java │ │ │ │ ├── MultiFieldQueryResult.java │ │ │ │ ├── QueryResult.java │ │ │ │ ├── TSDBConnection.java │ │ │ │ └── TSDBDump.java │ │ │ └── util/ │ │ │ ├── HttpUtils.java │ │ │ ├── TSDBUtils.java │ │ │ └── TimeUtils.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── reader/ │ └── tsdbreader/ │ ├── conn/ │ │ └── TSDBConnectionTest.java │ └── util/ │ ├── Const.java │ └── TimeUtilsTest.java ├── tsdbwriter/ │ ├── doc/ │ │ └── tsdbhttpwriter.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ └── package.xml │ │ ├── java/ │ │ │ └── com/ │ │ │ └── alibaba/ │ │ │ └── datax/ │ │ │ └── plugin/ │ │ │ └── writer/ │ │ │ ├── conn/ │ │ │ │ ├── Connection4TSDB.java │ │ │ │ ├── DataPoint4TSDB.java │ │ │ │ └── TSDBConnection.java │ │ │ ├── tsdbwriter/ │ │ │ │ ├── Constant.java │ │ │ │ ├── Key.java │ │ │ │ ├── SourceDBType.java │ │ │ │ ├── TSDBConverter.java │ │ │ │ ├── TSDBModel.java │ │ │ │ ├── TSDBWriter.java │ │ │ │ └── TSDBWriterErrorCode.java │ │ │ └── util/ │ │ │ ├── HttpUtils.java │ │ │ └── TSDBUtils.java │ │ └── resources/ │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test/ │ └── java/ │ └── com/ │ └── alibaba/ │ └── datax/ │ └── plugin/ │ └── writer/ │ ├── conn/ │ │ └── TSDBConnectionTest.java │ └── util/ │ ├── Const.java │ ├── HttpUtilsTest.java │ └── TSDBTest.java ├── txtfilereader/ │ ├── doc/ │ │ └── txtfilereader.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── reader/ │ │ └── txtfilereader/ │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── TxtFileReader.java │ │ └── TxtFileReaderErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json ├── txtfilewriter/ │ ├── doc/ │ │ └── txtfilewriter.md │ ├── pom.xml │ └── src/ │ └── main/ │ ├── assembly/ │ │ └── package.xml │ ├── java/ │ │ └── com/ │ │ └── alibaba/ │ │ └── datax/ │ │ └── plugin/ │ │ └── writer/ │ │ └── txtfilewriter/ │ │ ├── Key.java │ │ ├── TxtFileWriter.java │ │ └── TxtFileWriterErrorCode.java │ └── resources/ │ ├── plugin.json │ └── plugin_job_template.json └── userGuid.md
Showing preview only (616K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (6896 symbols across 864 files)
FILE: adbmysqlwriter/src/main/java/com/alibaba/datax/plugin/writer/adbmysqlwriter/AdbMysqlWriter.java
class AdbMysqlWriter (line 16) | public class AdbMysqlWriter extends Writer {
class Job (line 19) | public static class Job extends Writer.Job {
method preCheck (line 23) | @Override
method init (line 29) | @Override
method prepare (line 37) | @Override
method split (line 44) | @Override
method post (line 50) | @Override
method destroy (line 55) | @Override
class Task (line 62) | public static class Task extends Writer.Task {
class DelegateClass (line 67) | public static class DelegateClass extends CommonRdbmsWriter.Task {
method DelegateClass (line 72) | public DelegateClass(DataBaseType dataBaseType) {
method doBatchInsert (line 76) | @Override
method logTotalMetrics (line 93) | public void logTotalMetrics() {
method init (line 98) | @Override
method prepare (line 110) | @Override
method startWrite (line 116) | public void startWrite(RecordReceiver recordReceiver) {
method post (line 121) | @Override
method destroy (line 126) | @Override
method supportFailOver (line 131) | @Override
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/AdbpgWriter.java
class AdbpgWriter (line 26) | public class AdbpgWriter extends Writer {
class Job (line 29) | public static class Job extends Writer.Job {
method init (line 35) | @Override
method prepare (line 44) | @Override
method split (line 50) | @Override
method post (line 59) | @Override
method destroy (line 65) | @Override
class Task (line 74) | public static class Task extends Writer.Task {
method init (line 79) | @Override
method prepare (line 96) | @Override
method startWrite (line 101) | @Override
method post (line 106) | @Override
method destroy (line 111) | @Override
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/copy/Adb4pgClientProxy.java
class Adb4pgClientProxy (line 28) | public class Adb4pgClientProxy implements AdbProxy {
method Adb4pgClientProxy (line 38) | public Adb4pgClientProxy(Configuration configuration,TaskPluginCollect...
method startWriteWithConnection (line 113) | @Override
method prepareColumnTypeValue (line 161) | private void prepareColumnTypeValue(Record record, List<Object> values) {
method closeResource (line 173) | @Override
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/copy/AdbProxy.java
type AdbProxy (line 9) | public interface AdbProxy {
method startWriteWithConnection (line 10) | public abstract void startWriteWithConnection(RecordReceiver recordRec...
method closeResource (line 12) | public void closeResource();
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Adb4pgUtil.java
class Adb4pgUtil (line 28) | public class Adb4pgUtil {
method checkConfig (line 32) | public static void checkConfig(Configuration originalConfig) {
method convertConfiguration (line 43) | public static DatabaseConfig convertConfiguration(Configuration origin...
method splitBySchemaName (line 74) | private static Map<String, List<String>> splitBySchemaName(List<String...
method getAdbpgConnect (line 89) | public static Connection getAdbpgConnect(Configuration conf) {
method generateJdbcUrl (line 97) | private static String generateJdbcUrl(Configuration configuration) {
method prepare (line 105) | public static void prepare(Configuration originalConfig) {
method post (line 126) | public static void post(Configuration configuration) {
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Constant.java
class Constant (line 5) | public class Constant {
FILE: adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Key.java
class Key (line 5) | public class Key {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsException.java
class AdsException (line 3) | public class AdsException extends Exception {
method AdsException (line 29) | public AdsException(int code, String message, Throwable e) {
method getMessage (line 35) | @Override
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsWriter.java
class AdsWriter (line 41) | public class AdsWriter extends Writer {
class Job (line 43) | public static class Job extends Writer.Job {
method init (line 69) | @Override
method loadModeInit (line 95) | private void loadModeInit() {
method transferFromOdpsAndExit (line 164) | private void transferFromOdpsAndExit() {
method prepare (line 186) | @Override
method split (line 213) | @Override
method post (line 227) | @Override
method destroy (line 255) | @Override
method loadAdsData (line 264) | private void loadAdsData(AdsHelper helper, String odpsTableName, Str...
class Task (line 305) | public static class Task extends Writer.Task {
method init (line 321) | @Override
method prepare (line 359) | @Override
method startWrite (line 368) | public void startWrite(RecordReceiver recordReceiver) {
method post (line 388) | @Override
method destroy (line 397) | @Override
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsWriterErrorCode.java
type AdsWriterErrorCode (line 5) | public enum AdsWriterErrorCode implements ErrorCode {
method AdsWriterErrorCode (line 25) | private AdsWriterErrorCode(String code, String description) {
method setAdsAccount (line 30) | public void setAdsAccount(String adsAccount) {
method getCode (line 34) | @Override
method getDescription (line 39) | @Override
method toString (line 44) | @Override
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnDataType.java
class ColumnDataType (line 17) | public class ColumnDataType {
method getNames (line 50) | public static String getNames(int[] dataTypes) {
method add (line 96) | private static void add(int type, int sqlType, String jdbc, String[] n...
method getTypes (line 121) | public static ArrayList<ColumnDataType> getTypes() {
method getTypeClassName (line 131) | public static String getTypeClassName(int type) {
method getDataType (line 185) | public static ColumnDataType getDataType(int type) {
method convertTypeToSQLType (line 202) | public static int convertTypeToSQLType(int type) {
method convertSQLTypeToValueType (line 212) | public static int convertSQLTypeToValueType(int sqlType) {
method getTypeFromClass (line 260) | public static int getTypeFromClass(Class<?> x) {
method getNonPrimitiveClass (line 306) | public static Class<?> getNonPrimitiveClass(Class<?> clazz) {
method getTypeByName (line 337) | public static ColumnDataType getTypeByName(String s) {
method isStringType (line 347) | public static boolean isStringType(int type) {
method supportsAdd (line 358) | public boolean supportsAdd() {
method supportsAdd (line 368) | public static boolean supportsAdd(int type) {
method getAddProofType (line 389) | public static int getAddProofType(int type) {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java
class ColumnInfo (line 15) | public class ColumnInfo {
method getOrdinal (line 23) | public int getOrdinal() {
method setOrdinal (line 27) | public void setOrdinal(int ordinal) {
method getName (line 31) | public String getName() {
method setName (line 35) | public void setName(String name) {
method getDataType (line 39) | public ColumnDataType getDataType() {
method setDataType (line 43) | public void setDataType(ColumnDataType dataType) {
method isDeleted (line 47) | public boolean isDeleted() {
method setDeleted (line 51) | public void setDeleted(boolean isDeleted) {
method getComment (line 55) | public String getComment() {
method setComment (line 59) | public void setComment(String comment) {
method toString (line 63) | @Override
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java
class TableInfo (line 23) | public class TableInfo {
method toString (line 37) | @Override
method getTableSchema (line 47) | public String getTableSchema() {
method setTableSchema (line 51) | public void setTableSchema(String tableSchema) {
method getTableName (line 55) | public String getTableName() {
method setTableName (line 59) | public void setTableName(String tableName) {
method getColumns (line 63) | public List<ColumnInfo> getColumns() {
method getColumnsNames (line 67) | public List<String> getColumnsNames() {
method setColumns (line 75) | public void setColumns(List<ColumnInfo> columns) {
method getComments (line 79) | public String getComments() {
method setComments (line 83) | public void setComments(String comments) {
method getTableType (line 87) | public String getTableType() {
method setTableType (line 91) | public void setTableType(String tableType) {
method getUpdateType (line 95) | public String getUpdateType() {
method setUpdateType (line 99) | public void setUpdateType(String updateType) {
method getPartitionType (line 103) | public String getPartitionType() {
method setPartitionType (line 107) | public void setPartitionType(String partitionType) {
method getPartitionColumn (line 111) | public String getPartitionColumn() {
method setPartitionColumn (line 115) | public void setPartitionColumn(String partitionColumn) {
method getPartitionCount (line 119) | public int getPartitionCount() {
method setPartitionCount (line 123) | public void setPartitionCount(int partitionCount) {
method getPrimaryKeyColumns (line 127) | public List<String> getPrimaryKeyColumns() {
method setPrimaryKeyColumns (line 131) | public void setPrimaryKeyColumns(List<String> primaryKeyColumns) {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsClientProxy.java
class AdsClientProxy (line 31) | public class AdsClientProxy implements AdsProxy {
method AdsClientProxy (line 49) | public AdsClientProxy(String table, List<String> columns, Configuratio...
method startWriteWithConnection (line 155) | @Override
method prepareColumnTypeValue (line 202) | private void prepareColumnTypeValue(Record record, List<Object> values) {
method closeResource (line 213) | @Override
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java
class AdsInsertProxy (line 37) | public class AdsInsertProxy implements AdsProxy {
method AdsInsertProxy (line 69) | public AdsInsertProxy(String table, List<String> columns, Configuratio...
method startWriteWithConnection (line 119) | public void startWriteWithConnection(RecordReceiver recordReceiver,
method doBatchRecordWithPartitionSort (line 226) | private void doBatchRecordWithPartitionSort(List<Record> buffer, Strin...
method doBatchRecord (line 252) | private void doBatchRecord(final List<Record> buffer, final String mod...
method doBatchRecordDml (line 279) | @SuppressWarnings("resource")
method doOneRecord (line 334) | private void doOneRecord(List<Record> buffer, final String mode) {
method doOneRecordDml (line 354) | @SuppressWarnings("resource")
method isRetryable (line 401) | private boolean isRetryable(Throwable e) {
method generateDmlSql (line 412) | private String generateDmlSql(Connection connection, Record record, St...
method appendDmlSqlValues (line 477) | private void appendDmlSqlValues(Connection connection, Record record, ...
method prepareColumnTypeValue (line 489) | private void prepareColumnTypeValue(PreparedStatement statement, int c...
method getHashPartition (line 620) | private static int getHashPartition(String value, int totalHashPartiti...
method getCRC32 (line 625) | private static long getCRC32(String value) {
method closeResource (line 632) | @Override
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java
class AdsInsertUtil (line 28) | public class AdsInsertUtil {
method getAdsTableInfo (line 33) | public static TableInfo getAdsTableInfo(Configuration conf) {
method getAdsTableColumnNames (line 47) | public static List<String> getAdsTableColumnNames(Configuration conf) {
method getColumnMetaData (line 67) | public static Map<String, Pair<Integer,String>> getColumnMetaData
method getColumnMetaData (line 85) | public static Map<String, Pair<Integer,String>> getColumnMetaData(Tabl...
method getAdsTableColumns (line 105) | public static List<ColumnInfo> getAdsTableColumns(Configuration conf) {
method dealColumnConf (line 120) | public static void dealColumnConf(Configuration originalConfig, List<S...
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsProxy.java
type AdsProxy (line 7) | public interface AdsProxy {
method startWriteWithConnection (line 8) | public abstract void startWriteWithConnection(RecordReceiver recordRec...
method closeResource (line 11) | public void closeResource();
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/OperationType.java
type OperationType (line 3) | public enum OperationType {
method OperationType (line 7) | private OperationType(String type) {
method getType (line 13) | public String getType() {
method asOperationType (line 17) | public static OperationType asOperationType(String type) {
method isInsertTemplate (line 35) | public boolean isInsertTemplate() {
method isDeleteTemplate (line 51) | public boolean isDeleteTemplate() {
method isLegal (line 67) | public boolean isLegal() {
method toString (line 71) | @Override
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java
class AdsHelper (line 27) | public class AdsHelper {
method AdsHelper (line 38) | public AdsHelper(String adsUrl, String userName, String password, Stri...
method getAdsURL (line 47) | public String getAdsURL() {
method setAdsURL (line 51) | public void setAdsURL(String adsURL) {
method getUserName (line 55) | public String getUserName() {
method setUserName (line 59) | public void setUserName(String userName) {
method getPassword (line 63) | public String getPassword() {
method setPassword (line 67) | public void setPassword(String password) {
method getSchema (line 71) | public String getSchema() {
method setSchema (line 75) | public void setSchema(String schema) {
method getTableInfo (line 86) | public TableInfo getTableInfo(String table) throws AdsException {
method loadData (line 213) | public String loadData(String table, String partition, String sourcePa...
method checkLoadDataJobStatus (line 330) | public boolean checkLoadDataJobStatus(String jobId) throws AdsException {
method checkLoadDataJobStatusWithRetry (line 368) | private String checkLoadDataJobStatusWithRetry(final String jobId)
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java
class TableMetaHelper (line 19) | public class TableMetaHelper {
method TableMetaHelper (line 21) | private TableMetaHelper() {
method createTempODPSTable (line 31) | public static TableMeta createTempODPSTable(TableInfo tableMeta, int l...
method toODPSDataType (line 52) | private static String toODPSDataType(ColumnDataType columnDataType) {
method generateTempTableName (line 82) | private static String generateTempTableName(String tableSchema, String...
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java
class TransferProjectConf (line 8) | public class TransferProjectConf {
method create (line 24) | public static TransferProjectConf create(Configuration adsWriterConf) {
method getAccessId (line 35) | public String getAccessId() {
method getAccessKey (line 39) | public String getAccessKey() {
method getAccount (line 43) | public String getAccount() {
method getOdpsServer (line 47) | public String getOdpsServer() {
method getOdpsTunnel (line 51) | public String getOdpsTunnel() {
method getProject (line 56) | public String getProject() {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java
class DataType (line 18) | public class DataType {
method toString (line 26) | public static String toString(int type) {
method convertToDataType (line 60) | public static byte convertToDataType(String type) throws IllegalArgume...
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java
class FieldSchema (line 8) | public class FieldSchema {
method getName (line 18) | public String getName() {
method setName (line 22) | public void setName(String name) {
method getType (line 26) | public String getType() {
method setType (line 30) | public void setType(String type) {
method getComment (line 34) | public String getComment() {
method setComment (line 38) | public void setComment(String comment) {
method toString (line 42) | @Override
method toDDL (line 53) | public String toDDL() {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java
class TableMeta (line 11) | public class TableMeta {
method getTableName (line 23) | public String getTableName() {
method setTableName (line 27) | public void setTableName(String tableName) {
method getCols (line 31) | public List<FieldSchema> getCols() {
method setCols (line 35) | public void setCols(List<FieldSchema> cols) {
method getPartitionKeys (line 39) | public List<FieldSchema> getPartitionKeys() {
method setPartitionKeys (line 43) | public void setPartitionKeys(List<FieldSchema> partitionKeys) {
method getLifeCycle (line 47) | public int getLifeCycle() {
method setLifeCycle (line 51) | public void setLifeCycle(int lifeCycle) {
method getComment (line 55) | public String getComment() {
method setComment (line 59) | public void setComment(String comment) {
method toString (line 63) | @Override
method toDDL (line 81) | public String toDDL() {
method toDDL (line 104) | private String toDDL(List<FieldSchema> cols) {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java
class AdsUtil (line 21) | public class AdsUtil {
method checkNecessaryConfig (line 26) | public static void checkNecessaryConfig(Configuration originalConfig, ...
method createAdsHelper (line 57) | public static AdsHelper createAdsHelper(Configuration originalConfig){
method createAdsHelperWithOdpsAccount (line 68) | public static AdsHelper createAdsHelperWithOdpsAccount(Configuration o...
method generateConf (line 80) | public static Configuration generateConf(Configuration originalConfig,...
method generateSourcePath (line 104) | public static String generateSourcePath(String project, String tmpOdps...
method transferOdpsPartitionToAds (line 114) | public static String transferOdpsPartitionToAds(String odpsPartition){
method formatPartition (line 137) | public static String formatPartition(String partition) {
method prepareJdbcUrl (line 143) | public static String prepareJdbcUrl(Configuration conf) {
method prepareJdbcUrl (line 152) | public static String prepareJdbcUrl(String adsURL, String schema,
method getAdsConnect (line 168) | public static Connection getAdsConnect(Configuration conf) {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java
class Constant (line 3) | public class Constant {
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java
class Key (line 4) | public final class Key {
FILE: cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReader.java
class CassandraReader (line 19) | public class CassandraReader extends Reader {
class Job (line 23) | public static class Job extends Reader.Job {
method init (line 28) | @Override public void init() {
method destroy (line 51) | @Override public void destroy() {
method split (line 55) | @Override public List<Configuration> split(int adviceNumber) {
class Task (line 62) | public static class Task extends Reader.Task {
method init (line 71) | @Override public void init() {
method startRead (line 106) | @Override public void startRead(RecordSender recordSender) {
method destroy (line 117) | @Override public void destroy() {
FILE: cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java
type CassandraReaderErrorCode (line 5) | public enum CassandraReaderErrorCode implements ErrorCode {
method CassandraReaderErrorCode (line 12) | private CassandraReaderErrorCode(String code, String description) {
method getCode (line 17) | @Override
method getDescription (line 22) | @Override
method toString (line 27) | @Override
FILE: cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderHelper.java
class CassandraReaderHelper (line 49) | public class CassandraReaderHelper {
class TypeNotSupported (line 54) | static class TypeNotSupported extends Exception{}
method toJSonString (line 56) | static String toJSonString(Object o, DataType type ) throws Exception{
method transferObjectForJson (line 71) | static Object transferObjectForJson(Object o,DataType type) throws Typ...
method transferListForJson (line 138) | static List transferListForJson(List clist, DataType eleType) throws T...
method transferSetForJson (line 181) | static Set transferSetForJson(Set cset,DataType eleType) throws TypeNo...
method transferMapForJson (line 224) | static Map transferMapForJson(Map cmap,DataType keyType,DataType value...
method transferTupleForJson (line 239) | static List transferTupleForJson(TupleValue tupleValue,List<DataType> ...
method transferUDTForJson (line 250) | static Map transferUDTForJson(UDTValue udtValue) throws TypeNotSupport...
method buildRecord (line 263) | static Record buildRecord(Record record, Row rs, ColumnDefinitions met...
method splitJob (line 416) | public static List<Configuration> splitJob(int adviceNumber,Configurat...
method getQueryString (line 468) | public static String getQueryString(Configuration taskConfig,Cluster c...
method checkConfig (line 527) | public static void checkConfig(Configuration jobConfig,Cluster cluster) {
method ensureExists (line 565) | static void ensureExists(Configuration jobConfig,String keyword) {
method ensureStringExists (line 576) | static void ensureStringExists(Configuration jobConfig,String keyword) {
FILE: cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/Key.java
class Key (line 6) | public class Key {
FILE: cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriter.java
class CassandraWriter (line 37) | public class CassandraWriter extends Writer {
class Job (line 40) | public static class Job extends Writer.Job {
method split (line 43) | @Override public List<Configuration> split(int mandatoryNumber) {
method init (line 52) | @Override public void init() {
method destroy (line 56) | @Override public void destroy() {
class Task (line 61) | public static class Task extends Writer.Task {
method startWrite (line 75) | @Override public void startWrite(RecordReceiver lineReceiver) {
method init (line 156) | @Override public void init() {
method destroy (line 238) | @Override public void destroy() {
FILE: cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java
type CassandraWriterErrorCode (line 8) | public enum CassandraWriterErrorCode implements ErrorCode {
method CassandraWriterErrorCode (line 16) | private CassandraWriterErrorCode(String code, String description) {
method getCode (line 21) | @Override
method getDescription (line 26) | @Override
method toString (line 31) | @Override
FILE: cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterHelper.java
class CassandraWriterHelper (line 43) | public class CassandraWriterHelper {
method parseFromString (line 46) | public static Object parseFromString(String s, DataType sqlType ) thro...
method parseFromJson (line 145) | public static Object parseFromJson(Object jsonObject,DataType type) th...
method setupColumn (line 247) | public static void setupColumn(BoundStatement ps, int pos, DataType sq...
FILE: cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/Key.java
class Key (line 6) | public class Key {
FILE: clickhousereader/src/main/java/com/alibaba/datax/plugin/reader/clickhousereader/ClickhouseReader.java
class ClickhouseReader (line 24) | public class ClickhouseReader extends Reader {
class Job (line 29) | public static class Job extends Reader.Job {
method init (line 33) | @Override
method split (line 40) | @Override
method post (line 45) | @Override
method destroy (line 50) | @Override
class Task (line 56) | public static class Task extends Reader.Task {
method init (line 61) | @Override
method startRead (line 68) | @Override
method post (line 75) | @Override
method destroy (line 80) | @Override
FILE: clickhousereader/src/test/resources/basic1.sql
type default (line 1) | CREATE TABLE IF NOT EXISTS default.all_type_tbl
FILE: clickhousewriter/src/main/java/com/alibaba/datax/plugin/writer/clickhousewriter/ClickhouseWriter.java
class ClickhouseWriter (line 25) | public class ClickhouseWriter extends Writer {
class Job (line 28) | public static class Job extends Writer.Job {
method init (line 32) | @Override
method prepare (line 39) | @Override
method split (line 44) | @Override
method post (line 49) | @Override
method destroy (line 54) | @Override
class Task (line 60) | public static class Task extends Writer.Task {
method init (line 65) | @Override
method prepare (line 308) | @Override
method startWrite (line 313) | @Override
method post (line 318) | @Override
method destroy (line 323) | @Override
FILE: clickhousewriter/src/main/java/com/alibaba/datax/plugin/writer/clickhousewriter/ClickhouseWriterErrorCode.java
type ClickhouseWriterErrorCode (line 5) | public enum ClickhouseWriterErrorCode implements ErrorCode {
method ClickhouseWriterErrorCode (line 12) | private ClickhouseWriterErrorCode(String code, String description) {
method getCode (line 17) | @Override
method getDescription (line 22) | @Override
method toString (line 27) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/base/BaseObject.java
class BaseObject (line 8) | public class BaseObject {
method hashCode (line 10) | @Override
method equals (line 15) | @Override
method toString (line 20) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java
class CommonConstant (line 3) | public final class CommonConstant {
FILE: common/src/main/java/com/alibaba/datax/common/constant/PluginType.java
type PluginType (line 6) | public enum PluginType {
method PluginType (line 12) | private PluginType(String pluginType) {
method toString (line 16) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java
class BoolColumn (line 13) | public class BoolColumn extends Column {
method BoolColumn (line 15) | public BoolColumn(Boolean bool) {
method BoolColumn (line 19) | public BoolColumn(final String data) {
method BoolColumn (line 32) | public BoolColumn() {
method asBoolean (line 36) | @Override
method asLong (line 45) | @Override
method asDouble (line 54) | @Override
method asString (line 63) | @Override
method asBigInteger (line 72) | @Override
method asBigDecimal (line 81) | @Override
method asDate (line 90) | @Override
method asDate (line 96) | @Override
method asBytes (line 102) | @Override
method validate (line 108) | private void validate(final String data) {
FILE: common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java
class BytesColumn (line 14) | public class BytesColumn extends Column {
method BytesColumn (line 16) | public BytesColumn() {
method BytesColumn (line 20) | public BytesColumn(byte[] bytes) {
method asBytes (line 25) | @Override
method asString (line 34) | @Override
method asLong (line 49) | @Override
method asBigDecimal (line 55) | @Override
method asBigInteger (line 61) | @Override
method asDouble (line 67) | @Override
method asDate (line 73) | @Override
method asDate (line 79) | @Override
method asBoolean (line 85) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/element/Column.java
class Column (line 13) | public abstract class Column {
method Column (line 21) | public Column(final Object object, final Type type, int byteSize) {
method getRawData (line 27) | public Object getRawData() {
method getType (line 31) | public Type getType() {
method getByteSize (line 35) | public int getByteSize() {
method setType (line 39) | protected void setType(Type type) {
method setRawData (line 43) | protected void setRawData(Object rawData) {
method setByteSize (line 47) | protected void setByteSize(int byteSize) {
method asLong (line 51) | public abstract Long asLong();
method asDouble (line 53) | public abstract Double asDouble();
method asString (line 55) | public abstract String asString();
method asDate (line 57) | public abstract Date asDate();
method asDate (line 59) | public abstract Date asDate(String dateFormat);
method asBytes (line 61) | public abstract byte[] asBytes();
method asBoolean (line 63) | public abstract Boolean asBoolean();
method asBigDecimal (line 65) | public abstract BigDecimal asBigDecimal();
method asBigInteger (line 67) | public abstract BigInteger asBigInteger();
method toString (line 69) | @Override
type Type (line 74) | public enum Type {
FILE: common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java
class ColumnCast (line 13) | public final class ColumnCast {
method bind (line 15) | public static void bind(final Configuration configuration) {
method string2Date (line 21) | public static Date string2Date(final StringColumn column)
method string2Date (line 26) | public static Date string2Date(final StringColumn column, String dateF...
method string2Bytes (line 31) | public static byte[] string2Bytes(final StringColumn column)
method date2String (line 36) | public static String date2String(final DateColumn column) {
method bytes2String (line 40) | public static String bytes2String(final BytesColumn column)
class StringCast (line 46) | class StringCast {
method init (line 67) | static void init(final Configuration configuration) {
method asDate (line 92) | static Date asDate(final StringColumn column) throws ParseException {
method asDate (line 124) | static Date asDate(final StringColumn column, String dateFormat) throw...
method asBytes (line 134) | static byte[] asBytes(final StringColumn column)
class DateCast (line 149) | class DateCast {
method init (line 161) | static void init(final Configuration configuration) {
method asString (line 174) | static String asString(final DateColumn column) {
class BytesCast (line 197) | class BytesCast {
method init (line 200) | static void init(final Configuration configuration) {
method asString (line 206) | static String asString(final BytesColumn column)
FILE: common/src/main/java/com/alibaba/datax/common/element/DateColumn.java
class DateColumn (line 14) | public class DateColumn extends Column {
type DateType (line 22) | public static enum DateType {
method DateColumn (line 29) | public DateColumn(Time time, int nanos, int jdbcPrecision) {
method getNanos (line 42) | public long getNanos() {
method setNanos (line 46) | public void setNanos(int nanos) {
method getPrecision (line 50) | public int getPrecision() {
method setPrecision (line 54) | public void setPrecision(int precision) {
method DateColumn (line 61) | public DateColumn() {
method DateColumn (line 69) | public DateColumn(final Long stamp) {
method DateColumn (line 76) | public DateColumn(final Date date) {
method DateColumn (line 83) | public DateColumn(final java.sql.Date date) {
method DateColumn (line 91) | public DateColumn(final java.sql.Time time) {
method DateColumn (line 99) | public DateColumn(final java.sql.Timestamp ts) {
method asLong (line 104) | @Override
method asString (line 110) | @Override
method asDate (line 121) | @Override
method asDate (line 130) | @Override
method asBytes (line 135) | @Override
method asBoolean (line 141) | @Override
method asDouble (line 147) | @Override
method asBigInteger (line 153) | @Override
method asBigDecimal (line 159) | @Override
method getSubType (line 165) | public DateType getSubType() {
method setSubType (line 169) | public void setSubType(DateType subType) {
FILE: common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java
class DoubleColumn (line 10) | public class DoubleColumn extends Column {
method DoubleColumn (line 12) | public DoubleColumn(final String data) {
method DoubleColumn (line 17) | public DoubleColumn(Long data) {
method DoubleColumn (line 21) | public DoubleColumn(Integer data) {
method DoubleColumn (line 29) | public DoubleColumn(final Double data) {
method DoubleColumn (line 38) | public DoubleColumn(final Float data) {
method DoubleColumn (line 43) | public DoubleColumn(final BigDecimal data) {
method DoubleColumn (line 47) | public DoubleColumn(final BigInteger data) {
method DoubleColumn (line 51) | public DoubleColumn() {
method DoubleColumn (line 55) | private DoubleColumn(final String data, int byteSize) {
method asBigDecimal (line 59) | @Override
method asDouble (line 75) | @Override
method asLong (line 95) | @Override
method asBigInteger (line 107) | @Override
method asString (line 116) | @Override
method asBoolean (line 124) | @Override
method asDate (line 130) | @Override
method asDate (line 136) | @Override
method asBytes (line 142) | @Override
method validate (line 148) | private void validate(final String data) {
FILE: common/src/main/java/com/alibaba/datax/common/element/LongColumn.java
class LongColumn (line 11) | public class LongColumn extends Column {
method LongColumn (line 20) | public LongColumn(final String data) {
method LongColumn (line 41) | public LongColumn(Long data) {
method LongColumn (line 45) | public LongColumn(Integer data) {
method LongColumn (line 49) | public LongColumn(BigInteger data) {
method LongColumn (line 53) | private LongColumn(BigInteger data, int byteSize) {
method LongColumn (line 57) | public LongColumn() {
method asBigInteger (line 61) | @Override
method asLong (line 70) | @Override
method asDouble (line 82) | @Override
method asBoolean (line 94) | @Override
method asBigDecimal (line 104) | @Override
method asString (line 113) | @Override
method asDate (line 121) | @Override
method asDate (line 129) | @Override
method asBytes (line 134) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/element/OverFlowUtil.java
class OverFlowUtil (line 9) | public final class OverFlowUtil {
method isLongOverflow (line 22) | public static boolean isLongOverflow(final BigInteger integer) {
method validateLongNotOverFlow (line 28) | public static void validateLongNotOverFlow(final BigInteger integer) {
method isDoubleOverFlow (line 38) | public static boolean isDoubleOverFlow(final BigDecimal decimal) {
method validateDoubleNotOverFlow (line 53) | public static void validateDoubleNotOverFlow(final BigDecimal decimal) {
FILE: common/src/main/java/com/alibaba/datax/common/element/Record.java
type Record (line 9) | public interface Record {
method addColumn (line 11) | public void addColumn(Column column);
method setColumn (line 13) | public void setColumn(int i, final Column column);
method getColumn (line 15) | public Column getColumn(int i);
method toString (line 17) | public String toString();
method getColumnNumber (line 19) | public int getColumnNumber();
method getByteSize (line 21) | public int getByteSize();
method getMemorySize (line 23) | public int getMemorySize();
method setMeta (line 25) | public void setMeta(Map<String, String> meta);
method getMeta (line 27) | public Map<String, String> getMeta();
FILE: common/src/main/java/com/alibaba/datax/common/element/StringColumn.java
class StringColumn (line 14) | public class StringColumn extends Column {
method StringColumn (line 16) | public StringColumn() {
method StringColumn (line 20) | public StringColumn(final String rawData) {
method asString (line 25) | @Override
method validateDoubleSpecific (line 34) | private void validateDoubleSpecific(final String data) {
method asBigInteger (line 45) | @Override
method asLong (line 62) | @Override
method asBigDecimal (line 81) | @Override
method asDouble (line 98) | @Override
method asBoolean (line 123) | @Override
method asDate (line 142) | @Override
method asDate (line 153) | @Override
method asBytes (line 163) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/exception/CommonErrorCode.java
type CommonErrorCode (line 8) | public enum CommonErrorCode implements ErrorCode {
method CommonErrorCode (line 24) | private CommonErrorCode(String code, String describe) {
method getCode (line 29) | @Override
method getDescription (line 34) | @Override
method toString (line 39) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/exception/DataXException.java
class DataXException (line 8) | public class DataXException extends RuntimeException {
method DataXException (line 14) | public DataXException(ErrorCode errorCode, String errorMessage) {
method DataXException (line 19) | public DataXException(String errorMessage) {
method DataXException (line 23) | private DataXException(ErrorCode errorCode, String errorMessage, Throw...
method asDataXException (line 29) | public static DataXException asDataXException(ErrorCode errorCode, Str...
method asDataXException (line 33) | public static DataXException asDataXException(String message) {
method asDataXException (line 37) | public static DataXException asDataXException(ErrorCode errorCode, Str...
method asDataXException (line 44) | public static DataXException asDataXException(ErrorCode errorCode, Thr...
method getErrorCode (line 51) | public ErrorCode getErrorCode() {
method getMessage (line 55) | private static String getMessage(Object obj) {
FILE: common/src/main/java/com/alibaba/datax/common/exception/ExceptionTracker.java
class ExceptionTracker (line 6) | public final class ExceptionTracker {
method trace (line 9) | public static String trace(Throwable ex) {
FILE: common/src/main/java/com/alibaba/datax/common/plugin/AbstractJobPlugin.java
class AbstractJobPlugin (line 6) | public abstract class AbstractJobPlugin extends AbstractPlugin {
method getJobPluginCollector (line 10) | public JobPluginCollector getJobPluginCollector() {
method setJobPluginCollector (line 18) | public void setJobPluginCollector(
FILE: common/src/main/java/com/alibaba/datax/common/plugin/AbstractPlugin.java
class AbstractPlugin (line 8) | public abstract class AbstractPlugin extends BaseObject implements Plugi...
method getPluginName (line 22) | @Override
method getDeveloper (line 28) | @Override
method getDescription (line 34) | @Override
method getPluginJobConf (line 40) | @Override
method setPluginJobConf (line 45) | @Override
method setPluginConf (line 50) | @Override
method getPeerPluginJobConf (line 55) | @Override
method setPeerPluginJobConf (line 60) | @Override
method getPeerPluginName (line 65) | @Override
method setPeerPluginName (line 70) | @Override
method preCheck (line 75) | public void preCheck() {
method prepare (line 78) | public void prepare() {
method post (line 81) | public void post() {
method preHandler (line 84) | public void preHandler(Configuration jobConfiguration){
method postHandler (line 88) | public void postHandler(Configuration jobConfiguration){
method getReaderPluginSplitConf (line 92) | public List<Configuration> getReaderPluginSplitConf(){
method setReaderPluginSplitConf (line 96) | public void setReaderPluginSplitConf(List<Configuration> readerPluginS...
FILE: common/src/main/java/com/alibaba/datax/common/plugin/AbstractTaskPlugin.java
class AbstractTaskPlugin (line 6) | public abstract class AbstractTaskPlugin extends AbstractPlugin {
method getTaskPluginCollector (line 13) | public TaskPluginCollector getTaskPluginCollector() {
method setTaskPluginCollector (line 17) | public void setTaskPluginCollector(
method getTaskId (line 22) | public int getTaskId() {
method setTaskId (line 26) | public void setTaskId(int taskId) {
method getTaskGroupId (line 30) | public int getTaskGroupId() {
method setTaskGroupId (line 34) | public void setTaskGroupId(int taskGroupId) {
FILE: common/src/main/java/com/alibaba/datax/common/plugin/JobPluginCollector.java
type JobPluginCollector (line 9) | public interface JobPluginCollector extends PluginCollector {
method getMessage (line 15) | Map<String, List<String>> getMessage();
method getMessage (line 21) | List<String> getMessage(String key);
FILE: common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java
type PluginCollector (line 7) | public interface PluginCollector {
FILE: common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java
type Pluginable (line 5) | public interface Pluginable {
method getDeveloper (line 6) | String getDeveloper();
method getDescription (line 8) | String getDescription();
method setPluginConf (line 10) | void setPluginConf(Configuration pluginConf);
method init (line 12) | void init();
method destroy (line 14) | void destroy();
method getPluginName (line 16) | String getPluginName();
method getPluginJobConf (line 18) | Configuration getPluginJobConf();
method getPeerPluginJobConf (line 20) | Configuration getPeerPluginJobConf();
method getPeerPluginName (line 22) | public String getPeerPluginName();
method setPluginJobConf (line 24) | void setPluginJobConf(Configuration jobConf);
method setPeerPluginJobConf (line 26) | void setPeerPluginJobConf(Configuration peerPluginJobConf);
method setPeerPluginName (line 28) | public void setPeerPluginName(String peerPluginName);
FILE: common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java
type RecordReceiver (line 21) | public interface RecordReceiver {
method getFromReader (line 23) | public Record getFromReader();
method shutdown (line 25) | public void shutdown();
FILE: common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java
type RecordSender (line 21) | public interface RecordSender {
method createRecord (line 23) | public Record createRecord();
method sendToWriter (line 25) | public void sendToWriter(Record record);
method flush (line 27) | public void flush();
method terminate (line 29) | public void terminate();
method shutdown (line 31) | public void shutdown();
FILE: common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java
class TaskPluginCollector (line 13) | public abstract class TaskPluginCollector implements PluginCollector {
method collectDirtyRecord (line 24) | public abstract void collectDirtyRecord(final Record dirtyRecord,
method collectDirtyRecord (line 35) | public void collectDirtyRecord(final Record dirtyRecord,
method collectDirtyRecord (line 48) | public void collectDirtyRecord(final Record dirtyRecord, final Throwab...
method collectMessage (line 56) | public abstract void collectMessage(final String key, final String val...
FILE: common/src/main/java/com/alibaba/datax/common/spi/ErrorCode.java
type ErrorCode (line 15) | public interface ErrorCode {
method getCode (line 17) | String getCode();
method getDescription (line 20) | String getDescription();
method toString (line 32) | String toString();
FILE: common/src/main/java/com/alibaba/datax/common/spi/Hook.java
type Hook (line 10) | public interface Hook {
method getName (line 17) | public String getName();
method invoke (line 25) | public void invoke(Configuration jobConf, Map<String, Number> msg);
FILE: common/src/main/java/com/alibaba/datax/common/spi/Reader.java
class Reader (line 16) | public abstract class Reader extends BaseObject {
class Job (line 22) | public static abstract class Job extends AbstractJobPlugin {
method split (line 46) | public abstract List<Configuration> split(int adviceNumber);
class Task (line 49) | public static abstract class Task extends AbstractTaskPlugin {
method startRead (line 50) | public abstract void startRead(RecordSender recordSender);
FILE: common/src/main/java/com/alibaba/datax/common/spi/Writer.java
class Writer (line 16) | public abstract class Writer extends BaseObject {
class Job (line 20) | public abstract static class Job extends AbstractJobPlugin {
method split (line 28) | public abstract List<Configuration> split(int mandatoryNumber);
class Task (line 34) | public abstract static class Task extends AbstractTaskPlugin {
method startWrite (line 36) | public abstract void startWrite(RecordReceiver lineReceiver);
method supportFailOver (line 38) | public boolean supportFailOver(){return false;}
FILE: common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java
class PerfRecord (line 13) | @SuppressWarnings("NullableProblems")
type PHASE (line 19) | public enum PHASE {
method PHASE (line 59) | PHASE(int val) {
method toInt (line 63) | public int toInt(){
type ACTION (line 68) | public enum ACTION{
method PerfRecord (line 85) | public PerfRecord(int taskGroupId, int taskId, PHASE phase) {
method addPerfRecord (line 91) | public static void addPerfRecord(int taskGroupId, int taskId, PHASE ph...
method start (line 103) | public void start() {
method addCount (line 114) | public void addCount(long count) {
method addSize (line 118) | public void addSize(long size) {
method end (line 122) | public void end() {
method end (line 131) | public void end(long elapsedTimeInNs) {
method toString (line 140) | public String toString() {
method compareTo (line 147) | @Override
method hashCode (line 155) | @Override
method equals (line 166) | @Override
method copy (line 183) | public PerfRecord copy() {
method getTaskGroupId (line 192) | public int getTaskGroupId() {
method getTaskId (line 196) | public int getTaskId() {
method getPhase (line 200) | public PHASE getPhase() {
method getAction (line 204) | public ACTION getAction() {
method getElapsedTimeInNs (line 208) | public long getElapsedTimeInNs() {
method getCount (line 212) | public long getCount() {
method getSize (line 216) | public long getSize() {
method getInstId (line 220) | public long getInstId(){
method getHostIP (line 224) | public String getHostIP(){
method getHostName (line 228) | public String getHostName(){
method getStartTime (line 232) | public Date getStartTime() {
method getStartTimeInMs (line 236) | public long getStartTimeInMs() {
method getStartTimeInNs (line 240) | public long getStartTimeInNs() {
method getDatetime (line 244) | public String getDatetime(){
method isReport (line 251) | public boolean isReport() {
method setIsReport (line 255) | public void setIsReport(boolean isReport) {
FILE: common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java
class PerfTrace (line 20) | public class PerfTrace {
method getInstance (line 56) | public static PerfTrace getInstance(boolean isJob, long jobId, int tas...
method getInstance (line 73) | public static PerfTrace getInstance() {
method PerfTrace (line 85) | private PerfTrace(boolean isJob, long jobId, int taskGroupId, boolean ...
method addTaskDetails (line 100) | public void addTaskDetails(int taskId, String detail) {
method tracePerfRecord (line 120) | public void tracePerfRecord(PerfRecord perfRecord) {
method needReport (line 157) | private boolean needReport(PerfRecord perfRecord) {
method summarizeNoException (line 168) | public String summarizeNoException() {
method summarize (line 179) | private synchronized String summarize() {
method unitTime (line 238) | public static String unitTime(long time) {
method unitTime (line 242) | public static String unitTime(long time, TimeUnit timeUnit) {
method unitSize (line 246) | public static String unitSize(long size) {
method getPerfRecordMaps4print (line 259) | public synchronized ConcurrentHashMap<PHASE, SumPerfRecord4Print> getP...
method getSumPerf4Report (line 266) | public SumPerf4Report getSumPerf4Report() {
method getNeedReportPool4NotEnd (line 270) | public Set<PerfRecord> getNeedReportPool4NotEnd() {
method getTotalEndReport (line 274) | public List<PerfRecord> getTotalEndReport() {
method getTaskDetails (line 278) | public Map<Integer, String> getTaskDetails() {
method isEnable (line 282) | public boolean isEnable() {
method isJob (line 286) | public boolean isJob() {
method setJobInfo (line 300) | public void setJobInfo(Configuration jobInfo, boolean perfReportEnable...
method getWindow (line 329) | private Date getWindow(String windowStr, boolean startWindow) {
method getInstId (line 351) | public long getInstId() {
method getJobInfo (line 355) | public Configuration getJobInfo() {
method setBatchSize (line 359) | public void setBatchSize(int batchSize) {
method getReports (line 363) | public synchronized JobStatisticsDto2 getReports(String mode) {
method sumPerf4EndPrint (line 422) | private void sumPerf4EndPrint(List<PerfRecord> totalEndReport) {
method setChannelNumber (line 435) | public void setChannelNumber(int needChannelNumber) {
class SumPerf4Report (line 440) | public static class SumPerf4Report {
method add (line 446) | public void add(long curNanoTime,PerfRecord perfRecord) {
method getTotalTaskRunTimeInMs (line 473) | public long getTotalTaskRunTimeInMs() {
method getOdpsCloseTimeInMs (line 477) | public long getOdpsCloseTimeInMs() {
method getSqlQueryTimeInMs (line 481) | public long getSqlQueryTimeInMs() {
method getResultNextTimeInMs (line 485) | public long getResultNextTimeInMs() {
class SumPerfRecord4Print (line 490) | public static class SumPerfRecord4Print {
method add (line 507) | public void add(PerfRecord perfRecord) {
method getPerfTimeTotal (line 530) | public long getPerfTimeTotal() {
method getAverageTime (line 534) | public long getAverageTime() {
method getMaxTime (line 541) | public long getMaxTime() {
method getMaxTaskId (line 545) | public int getMaxTaskId() {
method getMaxTaskGroupId (line 549) | public int getMaxTaskGroupId() {
method getRecordsTotal (line 553) | public long getRecordsTotal() {
method getSizesTotal (line 557) | public long getSizesTotal() {
method getAverageRecords (line 561) | public long getAverageRecords() {
method getAverageBytes (line 568) | public long getAverageBytes() {
method getMaxRecord (line 575) | public long getMaxRecord() {
method getMaxByte (line 579) | public long getMaxByte() {
method getMaxTaskId4Records (line 583) | public int getMaxTaskId4Records() {
method getMaxTGID4Records (line 587) | public int getMaxTGID4Records() {
method getTotalCount (line 591) | public int getTotalCount() {
class JobStatisticsDto2 (line 595) | class JobStatisticsDto2 {
method getId (line 631) | public Long getId() {
method getGmtCreate (line 635) | public Date getGmtCreate() {
method getGmtModified (line 639) | public Date getGmtModified() {
method getInstId (line 643) | public Long getInstId() {
method getJobId (line 647) | public Long getJobId() {
method getJobVersion (line 651) | public Long getJobVersion() {
method getTaskGroupId (line 655) | public Integer getTaskGroupId() {
method getWindowStart (line 659) | public Date getWindowStart() {
method getWindowEnd (line 663) | public Date getWindowEnd() {
method getJobStartTime (line 667) | public Date getJobStartTime() {
method getJobEndTime (line 671) | public Date getJobEndTime() {
method getJobRunTimeMs (line 675) | public Long getJobRunTimeMs() {
method getChannelNum (line 679) | public Integer getChannelNum() {
method getCluster (line 683) | public String getCluster() {
method getJobDomain (line 687) | public String getJobDomain() {
method getSrcType (line 691) | public String getSrcType() {
method getDstType (line 695) | public String getDstType() {
method getSrcGuid (line 699) | public String getSrcGuid() {
method getDstGuid (line 703) | public String getDstGuid() {
method getRecords (line 707) | public Long getRecords() {
method getBytes (line 711) | public Long getBytes() {
method getSpeedRecord (line 715) | public Long getSpeedRecord() {
method getSpeedByte (line 719) | public Long getSpeedByte() {
method getStagePercent (line 723) | public String getStagePercent() {
method getErrorRecord (line 727) | public Long getErrorRecord() {
method getErrorBytes (line 731) | public Long getErrorBytes() {
method getWaitReadTimeMs (line 735) | public Long getWaitReadTimeMs() {
method getWaitWriteTimeMs (line 739) | public Long getWaitWriteTimeMs() {
method getOdpsBlockCloseTimeMs (line 743) | public Long getOdpsBlockCloseTimeMs() {
method getSqlQueryTimeMs (line 747) | public Long getSqlQueryTimeMs() {
method getResultNextTimeMs (line 751) | public Long getResultNextTimeMs() {
method getTaskTotalTimeMs (line 755) | public Long getTaskTotalTimeMs() {
method getHostAddress (line 759) | public String getHostAddress() {
method setId (line 763) | public void setId(Long id) {
method setGmtCreate (line 767) | public void setGmtCreate(Date gmtCreate) {
method setGmtModified (line 771) | public void setGmtModified(Date gmtModified) {
method setInstId (line 775) | public void setInstId(Long instId) {
method setJobId (line 779) | public void setJobId(Long jobId) {
method setJobVersion (line 783) | public void setJobVersion(Long jobVersion) {
method setTaskGroupId (line 787) | public void setTaskGroupId(Integer taskGroupId) {
method setWindowStart (line 791) | public void setWindowStart(Date windowStart) {
method setWindowEnd (line 795) | public void setWindowEnd(Date windowEnd) {
method setJobStartTime (line 799) | public void setJobStartTime(Date jobStartTime) {
method setJobEndTime (line 803) | public void setJobEndTime(Date jobEndTime) {
method setJobRunTimeMs (line 807) | public void setJobRunTimeMs(Long jobRunTimeMs) {
method setChannelNum (line 811) | public void setChannelNum(Integer channelNum) {
method setCluster (line 815) | public void setCluster(String cluster) {
method setJobDomain (line 819) | public void setJobDomain(String jobDomain) {
method setSrcType (line 823) | public void setSrcType(String srcType) {
method setDstType (line 827) | public void setDstType(String dstType) {
method setSrcGuid (line 831) | public void setSrcGuid(String srcGuid) {
method setDstGuid (line 835) | public void setDstGuid(String dstGuid) {
method setRecords (line 839) | public void setRecords(Long records) {
method setBytes (line 843) | public void setBytes(Long bytes) {
method setSpeedRecord (line 847) | public void setSpeedRecord(Long speedRecord) {
method setSpeedByte (line 851) | public void setSpeedByte(Long speedByte) {
method setStagePercent (line 855) | public void setStagePercent(String stagePercent) {
method setErrorRecord (line 859) | public void setErrorRecord(Long errorRecord) {
method setErrorBytes (line 863) | public void setErrorBytes(Long errorBytes) {
method setWaitReadTimeMs (line 867) | public void setWaitReadTimeMs(Long waitReadTimeMs) {
method setWaitWriteTimeMs (line 871) | public void setWaitWriteTimeMs(Long waitWriteTimeMs) {
method setOdpsBlockCloseTimeMs (line 875) | public void setOdpsBlockCloseTimeMs(Long odpsBlockCloseTimeMs) {
method setSqlQueryTimeMs (line 879) | public void setSqlQueryTimeMs(Long sqlQueryTimeMs) {
method setResultNextTimeMs (line 883) | public void setResultNextTimeMs(Long resultNextTimeMs) {
method setTaskTotalTimeMs (line 887) | public void setTaskTotalTimeMs(Long taskTotalTimeMs) {
method setHostAddress (line 891) | public void setHostAddress(String hostAddress) {
FILE: common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java
class VMInfo (line 18) | public class VMInfo {
method getVmInfo (line 28) | public static VMInfo getVmInfo() {
method VMInfo (line 73) | private VMInfo() {
method toString (line 115) | public String toString() {
method totalString (line 125) | public String totalString() {
method getDelta (line 129) | public void getDelta() {
method getDelta (line 133) | public synchronized void getDelta(boolean print) {
method isSunOsMBean (line 191) | public static boolean isSunOsMBean(OperatingSystemMXBean operatingSyst...
method getLongFromOperatingSystem (line 197) | public static long getLongFromOperatingSystem(OperatingSystemMXBean op...
class PhyOSStatus (line 209) | private class PhyOSStatus {
method toString (line 215) | public String toString() {
class ProcessCpuStatus (line 224) | private class ProcessCpuStatus {
method setMaxMinCpu (line 231) | public void setMaxMinCpu(float curCpu) {
method getDeltaString (line 242) | public String getDeltaString() {
method getTotalString (line 257) | public String getTotalString() {
class ProcessGCStatus (line 273) | private class ProcessGCStatus {
method toString (line 276) | public String toString() {
method getDeltaString (line 280) | public String getDeltaString() {
method getTotalString (line 298) | public String getTotalString() {
class ProcessMemoryStatus (line 316) | private class ProcessMemoryStatus {
method toString (line 319) | public String toString() {
method getDeltaString (line 331) | public String getDeltaString() {
class GCStatus (line 349) | private class GCStatus {
method setCurTotalGcCount (line 360) | public void setCurTotalGcCount(long curTotalGcCount) {
method setCurTotalGcTime (line 373) | public void setCurTotalGcTime(long curTotalGcTime) {
class MemoryStatus (line 387) | private class MemoryStatus {
method setMaxMinUsedSize (line 397) | void setMaxMinUsedSize(long curUsedSize) {
method setMaxMinPercent (line 404) | void setMaxMinPercent(float curPercent) {
FILE: common/src/main/java/com/alibaba/datax/common/util/Configuration.java
class Configuration (line 53) | public class Configuration {
method newDefault (line 67) | public static Configuration newDefault() {
method from (line 74) | public static Configuration from(String json) {
method from (line 90) | public static Configuration from(File file) {
method from (line 108) | public static Configuration from(InputStream is) {
method from (line 120) | public static Configuration from(final Map<String, Object> object) {
method from (line 127) | public static Configuration from(final List<Object> object) {
method getNecessaryValue (line 131) | public String getNecessaryValue(String key, ErrorCode errorCode) {
method getUnnecessaryValue (line 141) | public String getUnnecessaryValue(String key,String defaultValue,Error...
method getNecessaryBool (line 149) | public Boolean getNecessaryBool(String key, ErrorCode errorCode) {
method get (line 179) | public Object get(final String path) {
method getConfiguration (line 194) | public Configuration getConfiguration(final String path) {
method getString (line 208) | public String getString(final String path) {
method getString (line 221) | public String getString(final String path, final String defaultValue) {
method getChar (line 236) | public Character getChar(final String path) {
method getChar (line 257) | public Character getChar(final String path, char defaultValue) {
method getBool (line 270) | public Boolean getBool(final String path) {
method getBool (line 292) | public Boolean getBool(final String path, boolean defaultValue) {
method getInt (line 305) | public Integer getInt(final String path) {
method getInt (line 326) | public Integer getInt(final String path, int defaultValue) {
method getLong (line 339) | public Long getLong(final String path) {
method getLong (line 360) | public Long getLong(final String path, long defaultValue) {
method getDouble (line 373) | public Double getDouble(final String path) {
method getDouble (line 394) | public Double getDouble(final String path, double defaultValue) {
method getList (line 405) | @SuppressWarnings("unchecked")
method getListWithJson (line 414) | public <T> List<T> getListWithJson(final String path, Class<T> t) {
method getList (line 426) | @SuppressWarnings("unchecked")
method getList (line 446) | @SuppressWarnings("unchecked")
method getList (line 459) | public <T> List<T> getList(final String path, final List<T> defaultList,
method getListConfiguration (line 471) | public List<Configuration> getListConfiguration(final String path) {
method getMap (line 487) | @SuppressWarnings("unchecked")
method getMap (line 499) | @SuppressWarnings("unchecked")
method getMap (line 517) | @SuppressWarnings("unchecked")
method getMap (line 530) | public <T> Map<String, T> getMap(final String path,
method getMapConfiguration (line 542) | @SuppressWarnings("unchecked")
method get (line 578) | @SuppressWarnings("unchecked")
method beautify (line 587) | public String beautify() {
method set (line 613) | public Object set(final String path, final Object object) {
method getKeys (line 634) | public Set<String> getKeys() {
method remove (line 643) | public Object remove(final String path) {
method merge (line 664) | public Configuration merge(final Configuration another,
method toString (line 686) | @Override
method toJSON (line 694) | public String toJSON() {
method clone (line 701) | public Configuration clone() {
method addSecretKeyPath (line 715) | public void addSecretKeyPath(String path) {
method addSecretKeyPath (line 721) | public void addSecretKeyPath(Set<String> pathSet) {
method setSecretKeyPathSet (line 727) | public void setSecretKeyPathSet(Set<String> keyPathSet) {
method isSecretPath (line 733) | public boolean isSecretPath(String path) {
method getKeysRecursive (line 737) | @SuppressWarnings("unchecked")
method getInternal (line 772) | public Object getInternal() {
method setObject (line 776) | private void setObject(final String path, final Object object) {
method extractConfiguration (line 790) | @SuppressWarnings("unchecked")
method extractFromConfiguration (line 817) | private Object extractFromConfiguration(final Object object) {
method buildObject (line 825) | Object buildObject(final List<String> paths, final Object object) {
method setObjectRecursive (line 865) | @SuppressWarnings("unchecked")
method findObject (line 949) | private Object findObject(final String path) {
method findObjectInMap (line 970) | @SuppressWarnings("unchecked")
method findObjectInList (line 988) | @SuppressWarnings({ "unchecked" })
method expand (line 1008) | private List<Object> expand(List<Object> list, int size) {
method isPathList (line 1016) | private boolean isPathList(final String path) {
method isPathMap (line 1020) | private boolean isPathMap(final String path) {
method getIndex (line 1024) | private int getIndex(final String index) {
method isSuitForRoot (line 1028) | private boolean isSuitForRoot(final Object object) {
method split (line 1036) | private String split(final String path) {
method split2List (line 1040) | private List<String> split2List(final String path) {
method checkPath (line 1044) | private void checkPath(final String path) {
method toJSONPath (line 1058) | @SuppressWarnings("unused")
method checkJSON (line 1064) | private static void checkJSON(final String json) {
method Configuration (line 1071) | private Configuration(final String json) {
method toJSONString (line 1080) | private static String toJSONString(final Object object) {
method getSecretKeyPathSet (line 1084) | public Set<String> getSecretKeyPathSet() {
FILE: common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java
class ConfigurationUtil (line 9) | public class ConfigurationUtil {
method filterSensitive (line 13) | public static Configuration filterSensitive(Configuration origin) {
FILE: common/src/main/java/com/alibaba/datax/common/util/DESCipher.java
class DESCipher (line 33) | public class DESCipher {
method encrypt (line 48) | public static byte[] encrypt(byte[] src, byte[] key) throws Exception {
method decrypt (line 80) | public static byte[] decrypt(byte[] src, byte[] key) throws Exception {
method encrypt (line 109) | public static byte[] encrypt(byte[] src) throws Exception {
method decrypt (line 119) | public static byte[] decrypt(byte[] src) throws Exception {
method encrypt (line 129) | public final static String encrypt(String src) {
method encrypt (line 145) | public final static String encrypt(String src, String encryptKey) {
method decrypt (line 160) | public final static String decrypt(String src) {
method decrypt (line 176) | public final static String decrypt(String src, String decryptKey) {
method encryptToString (line 192) | public static String encryptToString(byte[] src) throws Exception {
method decryptToString (line 202) | public static String decryptToString(byte[] src) throws Exception {
method byte2hex (line 206) | public static String byte2hex(byte[] b) {
method hex2byte (line 219) | public static byte[] hex2byte(byte[] b) {
FILE: common/src/main/java/com/alibaba/datax/common/util/DataXCaseEnvUtil.java
class DataXCaseEnvUtil (line 7) | public class DataXCaseEnvUtil {
method getRetryTimes (line 16) | public static int getRetryTimes(int retryTimes) {
method getRetryInterval (line 22) | public static long getRetryInterval(long retryInterval) {
method getRetryExponential (line 28) | public static boolean getRetryExponential(boolean retryExponential) {
FILE: common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java
class FilterUtil (line 10) | public final class FilterUtil {
method filterByRegular (line 13) | public static List<String> filterByRegular(List<String> allStrs,
method filterByRegulars (line 34) | public static List<String> filterByRegulars(List<String> allStrs,
FILE: common/src/main/java/com/alibaba/datax/common/util/HostUtils.java
class HostUtils (line 13) | public class HostUtils {
FILE: common/src/main/java/com/alibaba/datax/common/util/LimitLogger.java
class LimitLogger (line 12) | public class LimitLogger {
method limit (line 16) | public static void limit(String name, long limit, LoggerFunction funct...
FILE: common/src/main/java/com/alibaba/datax/common/util/ListUtil.java
class ListUtil (line 16) | public final class ListUtil {
method checkIfValueDuplicate (line 18) | public static boolean checkIfValueDuplicate(List<String> aList,
method makeSureNoValueDuplicate (line 33) | public static void makeSureNoValueDuplicate(List<String> aList,
method checkIfBInA (line 65) | public static boolean checkIfBInA(List<String> aList, List<String> bList,
method makeSureBInA (line 80) | public static void makeSureBInA(List<String> aList, List<String> bList,
method checkIfValueSame (line 111) | public static boolean checkIfValueSame(List<Boolean> aList) {
method valueToLowerCase (line 129) | public static List<String> valueToLowerCase(List<String> aList) {
method checkIfHasSameValue (line 141) | public static Boolean checkIfHasSameValue(List<String> listA, List<Str...
method checkIfAllSameValue (line 155) | public static boolean checkIfAllSameValue(List<String> listA, List<Str...
FILE: common/src/main/java/com/alibaba/datax/common/util/LoggerFunction.java
type LoggerFunction (line 7) | public interface LoggerFunction {
method apply (line 9) | void apply();
FILE: common/src/main/java/com/alibaba/datax/common/util/MessageSource.java
class MessageSource (line 17) | public class MessageSource {
method MessageSource (line 24) | private MessageSource(ResourceBundle resourceBundle) {
method loadResourceBundle (line 36) | public static MessageSource loadResourceBundle(String baseName) {
method loadResourceBundle (line 45) | public static <T> MessageSource loadResourceBundle(Class<T> clazz) {
method loadResourceBundle (line 53) | public static <T> MessageSource loadResourceBundle(Class<T> clazz,
method loadResourceBundle (line 71) | public static MessageSource loadResourceBundle(String baseName,
method unloadResourceBundle (line 104) | public static <T> boolean unloadResourceBundle(Class<T> clazz) {
method reloadResourceBundle (line 115) | public static <T> MessageSource reloadResourceBundle(Class<T> clazz) {
method setEnvironment (line 120) | public static void setEnvironment(Locale locale, TimeZone timeZone) {
method init (line 129) | public static void init(final Configuration configuration) {
method clearCache (line 154) | public static void clearCache() {
method message (line 158) | public String message(String code) {
method message (line 162) | public String message(String code, String args1) {
method message (line 167) | public String message(String code, String args1, String args2) {
method message (line 172) | public String message(String code, String args1, String args2, String ...
method message (line 178) | public String message(String code, Object... args) {
method messageWithDefaultMessage (line 182) | public String messageWithDefaultMessage(String code, String defaultMes...
method messageWithDefaultMessage (line 191) | public String messageWithDefaultMessage(String code, String defaultMes...
FILE: common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java
class RangeSplitUtil (line 12) | public final class RangeSplitUtil {
method doAsciiStringSplit (line 14) | public static String[] doAsciiStringSplit(String left, String right, i...
method doLongSplit (line 33) | public static long[] doLongSplit(long left, long right, int expectSlic...
method doBigIntegerSplit (line 43) | public static BigInteger[] doBigIntegerSplit(BigInteger left, BigInteg...
method checkIfBetweenRange (line 95) | private static void checkIfBetweenRange(int value, int left, int right) {
method stringToBigInteger (line 105) | public static BigInteger stringToBigInteger(String aString, int radix) {
method bigIntegerToString (line 133) | private static String bigIntegerToString(BigInteger bigInteger, int ra...
method getMinAndMaxCharacter (line 177) | public static Pair<Character, Character> getMinAndMaxCharacter(String ...
method isPureAscii (line 195) | private static boolean isPureAscii(String aString) {
method doListSplit (line 213) | public static <T> List<List<T>> doListSplit(List<T> objects, int advic...
FILE: common/src/main/java/com/alibaba/datax/common/util/RetryUtil.java
class RetryUtil (line 9) | public final class RetryUtil {
method executeWithRetry (line 25) | public static <T> T executeWithRetry(Callable<T> callable,
method executeWithRetry (line 44) | public static <T> T executeWithRetry(Callable<T> callable,
method asyncExecuteWithRetry (line 68) | public static <T> T asyncExecuteWithRetry(Callable<T> callable,
method createThreadPoolExecutor (line 87) | public static ThreadPoolExecutor createThreadPoolExecutor() {
class Retry (line 94) | private static class Retry {
method doRetry (line 96) | public <T> T doRetry(Callable<T> callable, int retryTimes, long slee...
method call (line 163) | protected <T> T call(Callable<T> callable) throws Exception {
class AsyncRetry (line 168) | private static class AsyncRetry extends Retry {
method AsyncRetry (line 173) | public AsyncRetry(long timeoutMs, ThreadPoolExecutor executor) {
method call (line 191) | @Override
FILE: common/src/main/java/com/alibaba/datax/common/util/StrUtil.java
class StrUtil (line 14) | public class StrUtil {
method StrUtil (line 37) | private StrUtil() {
method stringify (line 40) | public static String stringify(long byteNumber) {
method replaceVariable (line 55) | public static String replaceVariable(final String param) {
method compressMiddle (line 76) | public static String compressMiddle(String s, int headLength, int tail...
method getMd5 (line 87) | public static String getMd5(String plainText) {
FILE: core/src/main/java/com/alibaba/datax/core/AbstractContainer.java
class AbstractContainer (line 10) | public abstract class AbstractContainer {
method AbstractContainer (line 15) | public AbstractContainer(Configuration configuration) {
method getConfiguration (line 21) | public Configuration getConfiguration() {
method getContainerCommunicator (line 25) | public AbstractContainerCommunicator getContainerCommunicator() {
method setContainerCommunicator (line 29) | public void setContainerCommunicator(AbstractContainerCommunicator con...
method start (line 33) | public abstract void start();
FILE: core/src/main/java/com/alibaba/datax/core/Engine.java
class Engine (line 34) | public class Engine {
method start (line 40) | public void start(Configuration allConf) {
method filterJobConfiguration (line 92) | public static String filterJobConfiguration(final Configuration config...
method filterSensitiveConfiguration (line 104) | public static Configuration filterSensitiveConfiguration(Configuration...
method entry (line 116) | public static void entry(final String[] args) throws Throwable {
method parseJobIdFromUrl (line 177) | private static long parseJobIdFromUrl(List<String> patternStringList, ...
method doParseJobIdFromUrl (line 188) | private static long doParseJobIdFromUrl(String patternString, String u...
method main (line 198) | public static void main(String[] args) throws Exception {
FILE: core/src/main/java/com/alibaba/datax/core/container/util/HookInvoker.java
class HookInvoker (line 28) | public class HookInvoker {
method HookInvoker (line 36) | public HookInvoker(String baseDirName, Configuration conf, Map<String,...
method invokeAll (line 42) | public void invokeAll() {
method doInvoke (line 65) | private void doInvoke(String path) {
method main (line 87) | public static void main(String[] args) {
FILE: core/src/main/java/com/alibaba/datax/core/container/util/JobAssignUtil.java
class JobAssignUtil (line 11) | public final class JobAssignUtil {
method JobAssignUtil (line 12) | private JobAssignUtil() {
method assignFairly (line 20) | public static List<Configuration> assignFairly(Configuration configura...
method adjustChannelNumPerTaskGroup (line 59) | private static void adjustChannelNumPerTaskGroup(List<Configuration> t...
method parseAndGetResourceMarkAndTaskIdMap (line 80) | private static LinkedHashMap<String, List<Integer>> parseAndGetResourc...
method doAssign (line 129) | private static List<Configuration> doAssign(LinkedHashMap<String, List...
FILE: core/src/main/java/com/alibaba/datax/core/job/JobContainer.java
class JobContainer (line 46) | public class JobContainer extends AbstractContainer {
method JobContainer (line 85) | public JobContainer(Configuration configuration) {
method start (line 95) | @Override
method preCheck (line 184) | private void preCheck() {
method preCheckInit (line 196) | private void preCheckInit() {
method preCheckReaderInit (line 215) | private Reader.Job preCheckReaderInit(JobPluginCollector jobPluginColl...
method preCheckWriterInit (line 240) | private Writer.Job preCheckWriterInit(JobPluginCollector jobPluginColl...
method preCheckReader (line 266) | private void preCheckReader() {
method preCheckWriter (line 275) | private void preCheckWriter() {
method init (line 287) | private void init() {
method prepare (line 307) | private void prepare() {
method preHandle (line 312) | private void preHandle() {
method postHandle (line 347) | private void postHandle() {
method split (line 386) | private int split() {
method adjustChannelNumber (line 416) | private void adjustChannelNumber() {
method schedule (line 492) | private void schedule() {
method initStandaloneScheduler (line 552) | private AbstractScheduler initStandaloneScheduler(Configuration config...
method post (line 559) | private void post() {
method destroy (line 564) | private void destroy() {
method logStatistics (line 575) | private void logStatistics() {
method initJobReader (line 654) | private Reader.Job initJobReader(
method initJobWriter (line 684) | private Writer.Job initJobWriter(
method prepareJobReader (line 710) | private void prepareJobReader() {
method prepareJobWriter (line 719) | private void prepareJobWriter() {
method doReaderSplit (line 729) | private List<Configuration> doReaderSplit(int adviceNumber) {
method doWriterSplit (line 745) | private List<Configuration> doWriterSplit(int readerTaskNumber) {
method mergeReaderAndWriterTaskConfigs (line 770) | private List<Configuration> mergeReaderAndWriterTaskConfigs(
method mergeReaderAndWriterTaskConfigs (line 776) | private List<Configuration> mergeReaderAndWriterTaskConfigs(
method distributeTasksToTaskGroup (line 836) | @SuppressWarnings("serial")
method postJobReader (line 939) | private void postJobReader() {
method postJobWriter (line 948) | private void postJobWriter() {
method checkLimit (line 962) | private void checkLimit() {
method invokeHooks (line 971) | private void invokeHooks() {
FILE: core/src/main/java/com/alibaba/datax/core/job/meta/ExecuteMode.java
type ExecuteMode (line 6) | public enum ExecuteMode {
method ExecuteMode (line 11) | private ExecuteMode(String value) {
method value (line 15) | public String value() {
method getValue (line 19) | public String getValue() {
FILE: core/src/main/java/com/alibaba/datax/core/job/meta/State.java
type State (line 6) | public enum State {
method State (line 17) | private State(int value) {
method value (line 21) | public int value() {
method isFinished (line 25) | public boolean isFinished() {
method isRunning (line 29) | public boolean isRunning() {
FILE: core/src/main/java/com/alibaba/datax/core/job/scheduler/AbstractScheduler.java
class AbstractScheduler (line 18) | public abstract class AbstractScheduler {
method getJobId (line 28) | public Long getJobId() {
method AbstractScheduler (line 32) | public AbstractScheduler(AbstractContainerCommunicator containerCommun...
method schedule (line 36) | public void schedule(List<Configuration> configurations) {
method startAllTaskGroup (line 114) | protected abstract void startAllTaskGroup(List<Configuration> configur...
method dealFailedStat (line 116) | protected abstract void dealFailedStat(AbstractContainerCommunicator f...
method dealKillingStat (line 118) | protected abstract void dealKillingStat(AbstractContainerCommunicator ...
method calculateTaskCount (line 120) | private int calculateTaskCount(List<Configuration> configurations) {
method isJobKilling (line 134) | protected abstract boolean isJobKilling(Long jobId);
FILE: core/src/main/java/com/alibaba/datax/core/job/scheduler/processinner/ProcessInnerScheduler.java
class ProcessInnerScheduler (line 15) | public abstract class ProcessInnerScheduler extends AbstractScheduler {
method ProcessInnerScheduler (line 19) | public ProcessInnerScheduler(AbstractContainerCommunicator containerCo...
method startAllTaskGroup (line 23) | @Override
method dealFailedStat (line 36) | @Override
method dealKillingStat (line 44) | @Override
method newTaskGroupContainerRunner (line 53) | private TaskGroupContainerRunner newTaskGroupContainerRunner(
FILE: core/src/main/java/com/alibaba/datax/core/job/scheduler/processinner/StandAloneScheduler.java
class StandAloneScheduler (line 8) | public class StandAloneScheduler extends ProcessInnerScheduler{
method StandAloneScheduler (line 10) | public StandAloneScheduler(AbstractContainerCommunicator containerComm...
method isJobKilling (line 14) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/communication/Communication.java
class Communication (line 17) | public class Communication extends BaseObject implements Cloneable {
method Communication (line 43) | public Communication() {
method reset (line 47) | public synchronized void reset() {
method init (line 51) | private void init() {
method getCounter (line 59) | public Map<String, Number> getCounter() {
method getState (line 63) | public State getState() {
method setState (line 67) | public synchronized void setState(State state, boolean isForce) {
method setState (line 75) | public synchronized void setState(State state) {
method getThrowable (line 79) | public Throwable getThrowable() {
method getThrowableMessage (line 83) | public synchronized String getThrowableMessage() {
method setThrowable (line 87) | public void setThrowable(Throwable throwable) {
method setThrowable (line 91) | public synchronized void setThrowable(Throwable throwable, boolean isF...
method getTimestamp (line 99) | public long getTimestamp() {
method setTimestamp (line 103) | public void setTimestamp(long timestamp) {
method getMessage (line 107) | public Map<String, List<String>> getMessage() {
method getMessage (line 111) | public List<String> getMessage(final String key) {
method addMessage (line 115) | public synchronized void addMessage(final String key, final String val...
method getLongCounter (line 126) | public synchronized Long getLongCounter(final String key) {
method setLongCounter (line 132) | public synchronized void setLongCounter(final String key, final long v...
method getDoubleCounter (line 137) | public synchronized Double getDoubleCounter(final String key) {
method setDoubleCounter (line 143) | public synchronized void setDoubleCounter(final String key, final doub...
method increaseCounter (line 148) | public synchronized void increaseCounter(final String key, final long ...
method clone (line 156) | @Override
method mergeFrom (line 195) | public synchronized Communication mergeFrom(final Communication otherC...
method mergeStateFrom (line 259) | public synchronized State mergeStateFrom(final Communication otherComm) {
method isFinished (line 276) | public synchronized boolean isFinished(){
FILE: core/src/main/java/com/alibaba/datax/core/statistics/communication/CommunicationTool.java
class CommunicationTool (line 15) | public final class CommunicationTool {
method getReportCommunication (line 52) | public static Communication getReportCommunication(Communication now, ...
method getTotalReadRecords (line 83) | public static long getTotalReadRecords(final Communication communicati...
method getTotalReadBytes (line 88) | public static long getTotalReadBytes(final Communication communication) {
method getTotalErrorRecords (line 93) | public static long getTotalErrorRecords(final Communication communicat...
method getTotalErrorBytes (line 98) | public static long getTotalErrorBytes(final Communication communicatio...
method getWriteSucceedRecords (line 103) | public static long getWriteSucceedRecords(final Communication communic...
method getWriteSucceedBytes (line 108) | public static long getWriteSucceedBytes(final Communication communicat...
class Stringify (line 113) | public static class Stringify {
method getSnapshot (line 116) | public static String getSnapshot(final Communication communication) {
method getTotal (line 155) | private static String getTotal(final Communication communication) {
method getSpeed (line 161) | private static String getSpeed(final Communication communication) {
method getError (line 167) | private static String getError(final Communication communication) {
method getPercentage (line 173) | private static String getPercentage(final Communication communicatio...
class Jsonify (line 178) | public static class Jsonify {
method getSnapshot (line 179) | @SuppressWarnings("rawtypes")
method getTotalBytes (line 221) | private static Pair<String, Long> getTotalBytes(final Communication ...
method getTotalRecords (line 225) | private static Pair<String, Long> getTotalRecords(final Communicatio...
method getSpeedByte (line 229) | private static Pair<String, Long> getSpeedByte(final Communication c...
method getSpeedRecord (line 233) | private static Pair<String, Long> getSpeedRecord(final Communication...
method getErrorRecords (line 237) | private static Pair<String, Long> getErrorRecords(final Communicatio...
method getErrorBytes (line 241) | private static Pair<String, Long> getErrorBytes(final Communication ...
method getStage (line 245) | private static Pair<String, Long> getStage(final Communication commu...
method getPercentage (line 249) | private static Pair<String, Double> getPercentage(final Communicatio...
method getErrorMessage (line 253) | private static Pair<String, String> getErrorMessage(final Communicat...
method getWaitReaderTime (line 257) | private static Pair<String, Long> getWaitReaderTime(final Communicat...
method getWaitWriterTime (line 261) | private static Pair<String, Long> getWaitWriterTime(final Communicat...
class Pair (line 265) | static class Pair<K, V> {
method Pair (line 266) | public Pair(final K key, final V value) {
method getKey (line 271) | public K getKey() {
method getValue (line 275) | public V getValue() {
FILE: core/src/main/java/com/alibaba/datax/core/statistics/communication/LocalTGCommunicationManager.java
class LocalTGCommunicationManager (line 10) | public final class LocalTGCommunicationManager {
method registerTaskGroupCommunication (line 14) | public static void registerTaskGroupCommunication(
method getJobCommunication (line 19) | public static Communication getJobCommunication() {
method getTaskGroupIdSet (line 37) | public static Set<Integer> getTaskGroupIdSet() {
method getTaskGroupCommunication (line 41) | public static Communication getTaskGroupCommunication(int taskGroupId) {
method updateTaskGroupCommunication (line 47) | public static void updateTaskGroupCommunication(final int taskGroupId,
method clear (line 55) | public static void clear() {
method getTaskGroupCommunicationMap (line 59) | public static Map<Integer, Communication> getTaskGroupCommunicationMap...
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/collector/AbstractCollector.java
class AbstractCollector (line 12) | public abstract class AbstractCollector {
method getTaskCommunicationMap (line 16) | public Map<Integer, Communication> getTaskCommunicationMap() {
method getJobId (line 20) | public Long getJobId() {
method setJobId (line 24) | public void setJobId(Long jobId) {
method registerTGCommunication (line 28) | public void registerTGCommunication(List<Configuration> taskGroupConfi...
method registerTaskCommunication (line 36) | public void registerTaskCommunication(List<Configuration> taskConfigur...
method collectFromTask (line 43) | public Communication collectFromTask() {
method collectFromTaskGroup (line 55) | public abstract Communication collectFromTaskGroup();
method getTGCommunicationMap (line 57) | public Map<Integer, Communication> getTGCommunicationMap() {
method getTGCommunication (line 61) | public Communication getTGCommunication(Integer taskGroupId) {
method getTaskCommunication (line 65) | public Communication getTaskCommunication(Integer taskId) {
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java
class ProcessInnerCollector (line 6) | public class ProcessInnerCollector extends AbstractCollector {
method ProcessInnerCollector (line 8) | public ProcessInnerCollector(Long jobId) {
method collectFromTaskGroup (line 12) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/AbstractContainerCommunicator.java
class AbstractContainerCommunicator (line 15) | public abstract class AbstractContainerCommunicator {
method getConfiguration (line 26) | public Configuration getConfiguration() {
method getCollector (line 30) | public AbstractCollector getCollector() {
method getReporter (line 34) | public AbstractReporter getReporter() {
method setCollector (line 38) | public void setCollector(AbstractCollector collector) {
method setReporter (line 42) | public void setReporter(AbstractReporter reporter) {
method getJobId (line 46) | public Long getJobId() {
method AbstractContainerCommunicator (line 50) | public AbstractContainerCommunicator(Configuration configuration) {
method registerCommunication (line 56) | public abstract void registerCommunication(List<Configuration> configu...
method collect (line 58) | public abstract Communication collect();
method report (line 60) | public abstract void report(Communication communication);
method collectState (line 62) | public abstract State collectState();
method getCommunication (line 64) | public abstract Communication getCommunication(Integer id);
method getCommunicationMap (line 70) | public abstract Map<Integer, Communication> getCommunicationMap();
method resetCommunication (line 72) | public void resetCommunication(Integer id){
method reportVmInfo (line 77) | public void reportVmInfo(){
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java
class StandAloneJobContainerCommunicator (line 17) | public class StandAloneJobContainerCommunicator extends AbstractContaine...
method StandAloneJobContainerCommunicator (line 21) | public StandAloneJobContainerCommunicator(Configuration configuration) {
method registerCommunication (line 28) | @Override
method collect (line 33) | @Override
method collectState (line 38) | @Override
method report (line 46) | @Override
method getCommunication (line 54) | @Override
method getCommunicationMap (line 59) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/AbstractTGContainerCommunicator.java
class AbstractTGContainerCommunicator (line 18) | public abstract class AbstractTGContainerCommunicator extends AbstractCo...
method AbstractTGContainerCommunicator (line 30) | public AbstractTGContainerCommunicator(Configuration configuration) {
method registerCommunication (line 39) | @Override
method collect (line 44) | @Override
method collectState (line 49) | @Override
method getCommunication (line 62) | @Override
method getCommunicationMap (line 69) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java
class StandaloneTGContainerCommunicator (line 7) | public class StandaloneTGContainerCommunicator extends AbstractTGContain...
method StandaloneTGContainerCommunicator (line 9) | public StandaloneTGContainerCommunicator(Configuration configuration) {
method report (line 14) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java
class AbstractReporter (line 5) | public abstract class AbstractReporter {
method reportJobCommunication (line 7) | public abstract void reportJobCommunication(Long jobId, Communication ...
method reportTGCommunication (line 9) | public abstract void reportTGCommunication(Integer taskGroupId, Commun...
FILE: core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java
class ProcessInnerReporter (line 6) | public class ProcessInnerReporter extends AbstractReporter {
method reportJobCommunication (line 8) | @Override
method reportTGCommunication (line 13) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/plugin/DefaultJobPluginCollector.java
class DefaultJobPluginCollector (line 13) | public final class DefaultJobPluginCollector implements JobPluginCollect...
method DefaultJobPluginCollector (line 16) | public DefaultJobPluginCollector(AbstractContainerCommunicator contain...
method getMessage (line 20) | @Override
method getMessage (line 26) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/AbstractTaskPluginCollector.java
class AbstractTaskPluginCollector (line 18) | public abstract class AbstractTaskPluginCollector extends TaskPluginColl...
method AbstractTaskPluginCollector (line 28) | public AbstractTaskPluginCollector(Configuration conf, Communication c...
method getCommunication (line 35) | public Communication getCommunication() {
method getConfiguration (line 39) | public Configuration getConfiguration() {
method getPluginType (line 43) | public PluginType getPluginType() {
method collectMessage (line 47) | @Override
method collectDirtyRecord (line 52) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/HttpPluginCollector.java
class HttpPluginCollector (line 11) | public class HttpPluginCollector extends AbstractTaskPluginCollector {
method HttpPluginCollector (line 12) | public HttpPluginCollector(Configuration configuration, Communication ...
method collectDirtyRecord (line 17) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/StdoutPluginCollector.java
class StdoutPluginCollector (line 22) | public class StdoutPluginCollector extends AbstractTaskPluginCollector {
method StdoutPluginCollector (line 32) | public StdoutPluginCollector(Configuration configuration, Communicatio...
method formatDirty (line 41) | private String formatDirty(final Record dirty, final Throwable t,
method collectDirtyRecord (line 60) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/util/DirtyRecord.java
class DirtyRecord (line 16) | public class DirtyRecord implements Record {
method asDirtyRecord (line 20) | public static DirtyRecord asDirtyRecord(final Record record) {
method addColumn (line 30) | @Override
method toString (line 36) | @Override
method setColumn (line 41) | @Override
method getColumn (line 47) | @Override
method getColumnNumber (line 53) | @Override
method getByteSize (line 59) | @Override
method getMemorySize (line 65) | @Override
method setMeta (line 71) | @Override
method getMeta (line 76) | @Override
method getColumns (line 81) | public List<Column> getColumns() {
method setColumns (line 85) | public void setColumns(List<Column> columns) {
class DirtyColumn (line 91) | class DirtyColumn extends Column {
method asDirtyColumn (line 94) | public static Column asDirtyColumn(final Column column, int index) {
method DirtyColumn (line 98) | private DirtyColumn(Column column, int index) {
method getIndex (line 104) | public int getIndex() {
method setIndex (line 108) | public void setIndex(int index) {
method asLong (line 112) | @Override
method asDouble (line 118) | @Override
method asString (line 124) | @Override
method asDate (line 130) | @Override
method asDate (line 136) | @Override
method asBytes (line 142) | @Override
method asBoolean (line 148) | @Override
method asBigDecimal (line 154) | @Override
method asBigInteger (line 160) | @Override
method DirtyColumn (line 166) | private DirtyColumn(Object object, Type type, int byteSize, int index) {
FILE: core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java
class TaskGroupContainer (line 37) | public class TaskGroupContainer extends AbstractContainer {
method TaskGroupContainer (line 63) | public TaskGroupContainer(Configuration configuration) {
method initCommunicator (line 79) | private void initCommunicator(Configuration configuration) {
method getJobId (line 84) | public long getJobId() {
method getTaskGroupId (line 88) | public int getTaskGroupId() {
method start (line 92) | @Override
method buildTaskConfigMap (line 300) | private Map<Integer, Configuration> buildTaskConfigMap(List<Configurat...
method buildRemainTasks (line 309) | private List<Configuration> buildRemainTasks(List<Configuration> confi...
method removeTask (line 317) | private TaskExecutor removeTask(List<TaskExecutor> taskList, int taskId){
method isAllTaskDone (line 329) | private boolean isAllTaskDone(List<TaskExecutor> taskList){
method reportTaskGroupCommunication (line 338) | private Communication reportTaskGroupCommunication(Communication lastT...
method markCommunicationFailed (line 347) | private void markCommunicationFailed(Integer taskId){
class TaskExecutor (line 356) | class TaskExecutor {
method TaskExecutor (line 381) | public TaskExecutor(Configuration taskConf, int attemptCount) {
method doStart (line 437) | public void doStart() {
method generateRunner (line 460) | private AbstractRunner generateRunner(PluginType pluginType) {
method generateRunner (line 464) | private AbstractRunner generateRunner(PluginType pluginType, List<Tr...
method isTaskFinished (line 523) | private boolean isTaskFinished() {
method getTaskId (line 536) | private int getTaskId(){
method getTimeStamp (line 540) | private long getTimeStamp(){
method getAttemptCount (line 544) | private int getAttemptCount(){
method supportFailOver (line 548) | private boolean supportFailOver(){
method shutdown (line 552) | private void shutdown(){
method isShutdown (line 563) | private boolean isShutdown(){
FILE: core/src/main/java/com/alibaba/datax/core/taskgroup/TaskMonitor.java
class TaskMonitor (line 16) | public class TaskMonitor {
method TaskMonitor (line 24) | private TaskMonitor() {
method getInstance (line 27) | public static TaskMonitor getInstance() {
method registerTask (line 31) | public void registerTask(Integer taskid, Communication communication) {
method removeTask (line 39) | public void removeTask(Integer taskid) {
method report (line 43) | public void report(Integer taskid, Communication communication) {
method getTaskCommunication (line 56) | public TaskCommunication getTaskCommunication(Integer taskid) {
class TaskCommunication (line 61) | public static class TaskCommunication {
method TaskCommunication (line 69) | private TaskCommunication(Integer taskid, Communication communicatio...
method report (line 76) | public void report(Communication communication) {
method isExpired (line 93) | private boolean isExpired(long lastUpdateComunicationTS) {
method getTaskid (line 97) | public Integer getTaskid() {
method getLastAllReadRecords (line 101) | public long getLastAllReadRecords() {
method getLastUpdateComunicationTS (line 105) | public long getLastUpdateComunicationTS() {
method getTtl (line 109) | public long getTtl() {
FILE: core/src/main/java/com/alibaba/datax/core/taskgroup/runner/AbstractRunner.java
class AbstractRunner (line 11) | public abstract class AbstractRunner {
method AbstractRunner (line 22) | public AbstractRunner(AbstractTaskPlugin taskPlugin) {
method destroy (line 26) | public void destroy() {
method getRunnerState (line 32) | public State getRunnerState() {
method getPlugin (line 36) | public AbstractTaskPlugin getPlugin() {
method setPlugin (line 40) | public void setPlugin(AbstractTaskPlugin plugin) {
method getJobConf (line 44) | public Configuration getJobConf() {
method setJobConf (line 48) | public void setJobConf(Configuration jobConf) {
method setTaskPluginCollector (line 53) | public void setTaskPluginCollector(TaskPluginCollector pluginCollector) {
method mark (line 57) | private void mark(State state) {
method markRun (line 66) | public void markRun() {
method markSuccess (line 70) | public void markSuccess() {
method markFail (line 74) | public void markFail(final Throwable throwable) {
method setTaskGroupId (line 83) | public void setTaskGroupId(int taskGroupId) {
method getTaskGroupId (line 91) | public int getTaskGroupId() {
method getTaskId (line 95) | public int getTaskId() {
method setTaskId (line 99) | public void setTaskId(int taskId) {
method setRunnerCommunication (line 104) | public void setRunnerCommunication(final Communication runnerCommunica...
method getRunnerCommunication (line 110) | public Communication getRunnerCommunication() {
method shutdown (line 114) | public abstract void shutdown();
FILE: core/src/main/java/com/alibaba/datax/core/taskgroup/runner/ReaderRunner.java
class ReaderRunner (line 16) | public class ReaderRunner extends AbstractRunner implements Runnable {
method setRecordSender (line 23) | public void setRecordSender(RecordSender recordSender) {
method ReaderRunner (line 27) | public ReaderRunner(AbstractTaskPlugin abstractTaskPlugin) {
method run (line 31) | @Override
method shutdown (line 92) | public void shutdown(){
FILE: core/src/main/java/com/alibaba/datax/core/taskgroup/runner/TaskGroupContainerRunner.java
class TaskGroupContainerRunner (line 8) | public class TaskGroupContainerRunner implements Runnable {
method TaskGroupContainerRunner (line 14) | public TaskGroupContainerRunner(TaskGroupContainer taskGroup) {
method run (line 19) | @Override
method getTaskGroupContainer (line 33) | public TaskGroupContainer getTaskGroupContainer() {
method getState (line 37) | public State getState() {
method setState (line 41) | public void setState(State state) {
FILE: core/src/main/java/com/alibaba/datax/core/taskgroup/runner/WriterRunner.java
class WriterRunner (line 17) | public class WriterRunner extends AbstractRunner implements Runnable {
method setRecordReceiver (line 24) | public void setRecordReceiver(RecordReceiver receiver) {
method WriterRunner (line 28) | public WriterRunner(AbstractTaskPlugin abstractTaskPlugin) {
method run (line 32) | @Override
method supportFailOver (line 82) | public boolean supportFailOver(){
method shutdown (line 87) | public void shutdown(){
FILE: core/src/main/java/com/alibaba/datax/core/transport/channel/Channel.java
class Channel (line 20) | public abstract class Channel {
method Channel (line 50) | public Channel(final Configuration configuration) {
method close (line 87) | public void close() {
method open (line 91) | public void open() {
method isClosed (line 95) | public boolean isClosed() {
method getTaskGroupId (line 99) | public int getTaskGroupId() {
method getCapacity (line 103) | public int getCapacity() {
method getByteSpeed (line 107) | public long getByteSpeed() {
method getConfiguration (line 111) | public Configuration getConfiguration() {
method setCommunication (line 115) | public void setCommunication(final Communication communication) {
method push (line 120) | public void push(final Record r) {
method pushTerminate (line 126) | public void pushTerminate(final TerminateRecord r) {
method pushAll (line 135) | public void pushAll(final Collection<Record> rs) {
method pull (line 142) | public Record pull() {
method pullAll (line 148) | public void pullAll(final Collection<Record> rs) {
method doPush (line 154) | protected abstract void doPush(Record r);
method doPushAll (line 156) | protected abstract void doPushAll(Collection<Record> rs);
method doPull (line 158) | protected abstract Record doPull();
method doPullAll (line 160) | protected abstract void doPullAll(Collection<Record> rs);
method size (line 162) | public abstract int size();
method isEmpty (line 164) | public abstract boolean isEmpty();
method clear (line 166) | public abstract void clear();
method getByteSize (line 168) | private long getByteSize(final Collection<Record> rs) {
method statPush (line 176) | private void statPush(long recordSize, long byteSize) {
method statPull (line 241) | private void statPull(long recordSize, long byteSize) {
FILE: core/src/main/java/com/alibaba/datax/core/transport/channel/memory/MemoryChannel.java
class MemoryChannel (line 22) | public class MemoryChannel extends Channel {
method MemoryChannel (line 34) | public MemoryChannel(final Configuration configuration) {
method close (line 44) | @Override
method clear (line 54) | @Override
method doPush (line 59) | @Override
method doPushAll (line 71) | @Override
method doPull (line 92) | @Override
method doPullAll (line 106) | @Override
method getRecordBytes (line 128) | private int getRecordBytes(Collection<Record> rs){
method size (line 136) | @Override
method isEmpty (line 141) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/transport/exchanger/BufferedRecordExchanger.java
class BufferedRecordExchanger (line 20) | public class BufferedRecordExchanger implements RecordSender, RecordRece...
method BufferedRecordExchanger (line 42) | @SuppressWarnings("unchecked")
method createRecord (line 70) | @Override
method sendToWriter (line 80) | @Override
method flush (line 103) | @Override
method terminate (line 114) | @Override
method getFromReader (line 123) | @Override
method shutdown (line 140) | @Override
method receive (line 151) | private void receive() {
FILE: core/src/main/java/com/alibaba/datax/core/transport/exchanger/BufferedRecordTransformerExchanger.java
class BufferedRecordTransformerExchanger (line 22) | public class BufferedRecordTransformerExchanger extends TransformerExcha...
method BufferedRecordTransformerExchanger (line 43) | @SuppressWarnings("unchecked")
method createRecord (line 74) | @Override
method sendToWriter (line 84) | @Override
method flush (line 113) | @Override
method terminate (line 126) | @Override
method getFromReader (line 135) | @Override
method shutdown (line 152) | @Override
method receive (line 163) | private void receive() {
FILE: core/src/main/java/com/alibaba/datax/core/transport/exchanger/RecordExchanger.java
class RecordExchanger (line 35) | public class RecordExchanger extends TransformerExchanger implements Rec...
method RecordExchanger (line 45) | @SuppressWarnings("unchecked")
method getFromReader (line 62) | @Override
method createRecord (line 71) | @Override
method sendToWriter (line 81) | @Override
method flush (line 95) | @Override
method terminate (line 99) | @Override
method shutdown (line 109) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/transport/exchanger/TransformerExchanger.java
class TransformerExchanger (line 20) | public abstract class TransformerExchanger {
method TransformerExchanger (line 41) | public TransformerExchanger(int taskGroupId, int taskId, Communication...
method doTransformer (line 53) | public Record doTransformer(Record record) {
method doStat (line 129) | public void doStat() {
FILE: core/src/main/java/com/alibaba/datax/core/transport/record/DefaultRecord.java
class DefaultRecord (line 19) | public class DefaultRecord implements Record {
method DefaultRecord (line 32) | public DefaultRecord() {
method addColumn (line 36) | @Override
method getColumn (line 42) | @Override
method setColumn (line 50) | @Override
method toString (line 66) | @Override
method getColumnNumber (line 74) | @Override
method getByteSize (line 79) | @Override
method getMemorySize (line 84) | public int getMemorySize(){
method setMeta (line 88) | @Override
method getMeta (line 93) | @Override
method decrByteSize (line 98) | private void decrByteSize(final Column column) {
method incrByteSize (line 109) | private void incrByteSize(final Column column) {
method expandCapacity (line 120) | private void expandCapacity(int totalSize) {
FILE: core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java
class TerminateRecord (line 12) | public class TerminateRecord implements Record {
method TerminateRecord (line 15) | private TerminateRecord() {
method get (line 18) | public static TerminateRecord get() {
method addColumn (line 22) | @Override
method getColumn (line 26) | @Override
method getColumnNumber (line 31) | @Override
method getByteSize (line 36) | @Override
method getMemorySize (line 41) | @Override
method setMeta (line 46) | @Override
method getMeta (line 51) | @Override
method setColumn (line 56) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/ComplexTransformerProxy.java
class ComplexTransformerProxy (line 13) | public class ComplexTransformerProxy extends ComplexTransformer {
method ComplexTransformerProxy (line 16) | public ComplexTransformerProxy(Transformer transformer) {
method evaluate (line 21) | @Override
method getRealTransformer (line 26) | public Transformer getRealTransformer() {
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/DigestTransformer.java
class DigestTransformer (line 20) | public class DigestTransformer extends Transformer {
method DigestTransformer (line 27) | public DigestTransformer() {
method evaluate (line 31) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java
class FilterTransformer (line 14) | public class FilterTransformer extends Transformer {
method FilterTransformer (line 15) | public FilterTransformer() {
method evaluate (line 19) | @Override
method doGreat (line 72) | private Record doGreat(Record record, String value, Column column, boo...
method doLess (line 132) | private Record doLess(Record record, String value, Column column, bool...
method doEqual (line 203) | private Record doEqual(Record record, String value, Column column) {
method doNotEqual (line 253) | private Record doNotEqual(Record record, String value, Column column) {
method doLike (line 294) | private Record doLike(Record record, String value, Column column) {
method doNotLike (line 303) | private Record doNotLike(Record record, String value, Column column) {
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformer.java
class GroovyTransformer (line 17) | public class GroovyTransformer extends Transformer {
method GroovyTransformer (line 18) | public GroovyTransformer() {
method evaluate (line 24) | @Override
method initGroovyTransformer (line 45) | private void initGroovyTransformer(String code, List<String> extraPack...
method getGroovyRule (line 68) | private String getGroovyRule(String expression, List<String> extraPack...
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java
class GroovyTransformerStaticUtil (line 9) | public class GroovyTransformerStaticUtil {
method md5 (line 11) | public static String md5(final String data) {
method sha1 (line 15) | public static String sha1(final String data) {
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/PadTransformer.java
class PadTransformer (line 15) | public class PadTransformer extends Transformer {
method PadTransformer (line 16) | public PadTransformer() {
method evaluate (line 20) | @Override
method doPad (line 69) | private String doPad(String padType, String oriValue, int length, Stri...
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/ReplaceTransformer.java
class ReplaceTransformer (line 15) | public class ReplaceTransformer extends Transformer {
method ReplaceTransformer (line 16) | public ReplaceTransformer() {
method evaluate (line 20) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/SubstrTransformer.java
class SubstrTransformer (line 15) | public class SubstrTransformer extends Transformer {
method SubstrTransformer (line 16) | public SubstrTransformer() {
method evaluate (line 20) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerErrorCode.java
type TransformerErrorCode (line 5) | public enum TransformerErrorCode implements ErrorCode {
method TransformerErrorCode (line 20) | private TransformerErrorCode(String code, String description) {
method getCode (line 25) | @Override
method getDescription (line 30) | @Override
method toString (line 35) | @Override
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerExecution.java
class TransformerExecution (line 11) | public class TransformerExecution {
method TransformerExecution (line 19) | public TransformerExecution(TransformerInfo transformerInfo ,Transform...
method genFinalParas (line 38) | public void genFinalParas() {
method getFinalParas (line 72) | public Object[] getFinalParas() {
method getExaustedTime (line 76) | public long getExaustedTime() {
method getSuccessRecords (line 80) | public long getSuccessRecords() {
method getFailedRecords (line 84) | public long getFailedRecords() {
method getFilterRecords (line 88) | public long getFilterRecords() {
method setIsChecked (line 92) | public void setIsChecked(boolean isChecked) {
method isChecked (line 96) | public boolean isChecked() {
method getClassLoader (line 103) | public ClassLoader getClassLoader() {
method getColumnIndex (line 107) | public Integer getColumnIndex() {
method getTransformerName (line 111) | public String getTransformerName() {
method getTransformer (line 115) | public ComplexTransformer getTransformer() {
method gettContext (line 119) | public Map<String,Object> gettContext() {
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerExecutionParas.java
class TransformerExecutionParas (line 10) | public class TransformerExecutionParas {
method getColumnIndex (line 23) | public Integer getColumnIndex() {
method getParas (line 27) | public String[] getParas() {
method gettContext (line 31) | public Map<String, Object> gettContext() {
method getCode (line 35) | public String getCode() {
method getExtraPackage (line 39) | public List<String> getExtraPackage() {
method setColumnIndex (line 43) | public void setColumnIndex(Integer columnIndex) {
method setParas (line 47) | public void setParas(String[] paras) {
method settContext (line 51) | public void settContext(Map<String, Object> tContext) {
method setCode (line 55) | public void setCode(String code) {
method setExtraPackage (line 59) | public void setExtraPackage(List<String> extraPackage) {
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerInfo.java
class TransformerInfo (line 9) | public class TransformerInfo {
method getTransformer (line 19) | public ComplexTransformer getTransformer() {
method getClassLoader (line 23) | public ClassLoader getClassLoader() {
method isNative (line 27) | public boolean isNative() {
method setTransformer (line 31) | public void setTransformer(ComplexTransformer transformer) {
method setClassLoader (line 35) | public void setClassLoader(ClassLoader classLoader) {
method setIsNative (line 39) | public void setIsNative(boolean isNative) {
FILE: core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerRegistry.java
class TransformerRegistry (line 23) | public class TransformerRegistry {
method loadTransformerFromLocalStorage (line 42) | public static void loadTransformerFromLocalStorage() {
method loadTransformerFromLocalStorage (line 48) | public static void loadTransformerFromLocalStorage(List<String> transf...
method loadTransformer (line 67) | public static void loadTransformer(String each) {
method loadTransFormerConfig (line 107) | private static Configuration loadTransFormerConfig(String transformerP...
method getTransformer (line 111) | public static TransformerInfo getTransformer(String transformerName) {
method registTransformer (line 122) | public static synchronized void registTransformer(Transformer transfor...
method registTransformer (line 126) | public static synchronized void registTransformer(Transformer transfor...
method registComplexTransformer (line 138) | public static synchronized void registComplexTransformer(ComplexTransf...
method checkName (line 149) | private static void checkName(String functionName, boolean isNative) {
method buildTransformerInfo (line 167) | private static TransformerInfo buildTransformerInfo(ComplexTransformer...
method getAllSuportTransformer (line 175) | public static List<String> getAllSuportTransformer() {
FILE: core/src/main/java/com/alibaba/datax/core/util/ClassSize.java
class ClassSize (line 6) | public class ClassSize {
method align (line 33) | public static int align(int num) {
method align (line 37) | public static long align(long num) {
FILE: core/src/main/java/com/alibaba/datax/core/util/ClassUtil.java
class ClassUtil (line 5) | public final class ClassUtil {
method instantiate (line 18) | @SuppressWarnings({ "rawtypes", "unchecked" })
method toClassType (line 30) | private static Class<?>[] toClassType(Object[] args) {
FILE: core/src/main/java/com/alibaba/datax/core/util/ConfigParser.java
class ConfigParser (line 20) | public final class ConfigParser {
method parse (line 25) | public static Configuration parse(final String jobPath) {
method parseCoreConfig (line 69) | private static Configuration parseCoreConfig(final String path) {
method parseJobConfig (line 73) | public static Configuration parseJobConfig(final String path) {
method getJobContent (line 80) | private static String getJobContent(String jobResource) {
method parsePluginConfig (line 118) | public static Configuration parsePluginConfig(List<String> wantPluginN...
method parseOnePluginConfig (line 149) | public static Configuration parseOnePluginConfig(final String path,
method getDirAsList (line 183) | private static List<String> getDirAsList(String path) {
FILE: core/src/main/java/com/alibaba/datax/core/util/ConfigurationValidate.java
class ConfigurationValidate (line 11) | public class ConfigurationValidate {
method doValidate (line 12) | public static void doValidate(Configuration allConfig) {
method coreValidate (line 22) | private static void coreValidate(Configuration allconfig) {
method pluginValidate (line 26) | private static void pluginValidate(Configuration allConfig) {
method jobValidate (line 30) | private static void jobValidate(Configuration allConfig) {
FILE: core/src/main/java/com/alibaba/datax/core/util/ErrorRecordChecker.java
class ErrorRecordChecker (line 18) | public final class ErrorRecordChecker {
method ErrorRecordChecker (line 25) | public ErrorRecordChecker(Configuration configuration) {
method ErrorRecordChecker (line 30) | public ErrorRecordChecker(Long rec, Double percentage) {
method checkRecordLimit (line 48) | public void checkRecordLimit(Communication communication) {
method checkPercentageLimit (line 65) | public void checkPercentageLimit(Communication communication) {
FILE: core/src/main/java/com/alibaba/datax/core/util/ExceptionTracker.java
class ExceptionTracker (line 6) | public class ExceptionTracker {
method trace (line 9) | public static String trace(Throwable ex) {
FILE: core/src/main/java/com/alibaba/datax/core/util/FrameworkErrorCode.java
type FrameworkErrorCode (line 10) | public enum FrameworkErrorCode implements ErrorCode {
method FrameworkErrorCode (line 36) | private FrameworkErrorCode(String code, String description) {
method getCode (line 41) | @Override
method getDescription (line 46) | @Override
method toString (line 51) | @Override
method toExitValue (line 60) | public int toExitValue() {
FILE: core/src/main/java/com/alibaba/datax/core/util/HttpClientUtil.java
class HttpClientUtil (line 25) | public class HttpClientUtil {
method setHttpTimeoutInMillionSeconds (line 40) | public static void setHttpTimeoutInMillionSeconds(int httpTimeoutInMil...
method getHttpClientUtil (line 44) | public static synchronized HttpClientUtil getHttpClientUtil() {
method HttpClientUtil (line 55) | public HttpClientUtil() {
method destroy (line 61) | public void destroy() {
method setBasicAuth (line 65) | public static void setBasicAuth(String username, String password) {
method initApacheHttpClient (line 72) | private void initApacheHttpClient() {
method destroyApacheHttpClient (line 86) | private void destroyApacheHttpClient() {
method getGetRequest (line 97) | public static HttpGet getGetRequest() {
method getPostRequest (line 101) | public static HttpPost getPostRequest() {
method getPutRequest (line 105) | public static HttpPut getPutRequest() {
method getDeleteRequest (line 109) | public static HttpDelete getDeleteRequest() {
method executeAndGet (line 113) | public String executeAndGet(HttpRequestBase httpRequestBase) throws Ex...
method executeAndGetWithRetry (line 141) | public String executeAndGetWithRetry(final HttpRequestBase httpRequest...
method executeAndGetWithFailedRetry (line 154) | public String executeAndGetWithFailedRetry(final HttpRequestBase httpR...
FILE: core/src/main/java/com/alibaba/datax/core/util/SecretUtil.java
class SecretUtil (line 35) | public class SecretUtil {
method encryptBASE64 (line 59) | public static String encryptBASE64(byte[] plaintextBytes) throws Excep...
method decryptBASE64 (line 70) | public static byte[] decryptBASE64(String cipherText) {
method encrypt (line 79) | public static String encrypt(String data, String key, String method) {
method decrypt (line 96) | public static String decrypt(String data, String key, String method) {
method encryptRSA (line 117) | public static String encryptRSA(String data, String key) {
method decryptRSA (line 147) | public static String decryptRSA(String data, String key) {
method initKey (line 174) | public static String[] initKey() throws Exception {
method encrypt3DES (line 203) | public static String encrypt3DES(String data, String key) {
method decrypt3DES (line 227) | public static String decrypt3DES(String data, String key) {
method build3DesKey (line 249) | private static byte[] build3DesKey(String keyStr) {
method getSecurityProperties (line 268) | public static synchronized Properties getSecurityProperties() {
method encryptSecretKey (line 294) | public static Configuration encryptSecretKey(Configuration configurati...
method decryptSecretKey (line 334) | public static Configuration decryptSecretKey(Configuration config) {
method getPrivateKeyMap (line 378) | private static synchronized Map<String, Triple<String, String, String>...
FILE: core/src/main/java/com/alibaba/datax/core/util/TransformerUtil.java
class TransformerUtil (line 18) | public class TransformerUtil {
method buildTransformerInfo (line 22) | public static List<TransformerExecution> buildTransformerInfo(Configur...
FILE: core/src/main/java/com/alibaba/datax/core/util/container/ClassLoaderSwapper.java
class ClassLoaderSwapper (line 9) | public final class ClassLoaderSwapper {
method ClassLoaderSwapper (line 12) | private ClassLoaderSwapper() {
method newCurrentThreadClassLoaderSwapper (line 15) | public static ClassLoaderSwapper newCurrentThreadClassLoaderSwapper() {
method setCurrentThreadClassLoader (line 25) | public ClassLoader setCurrentThreadClassLoader(ClassLoader classLoader) {
method restoreCurrentThreadClassLoader (line 35) | public ClassLoader restoreCurrentThreadClassLoader() {
FILE: core/src/main/java/com/alibaba/datax/core/util/container/CoreConstant.java
class CoreConstant (line 10) | public class CoreConstant {
FILE: core/src/main/java/com/alibaba/datax/core/util/container/JarLoader.java
class JarLoader (line 18) | public class JarLoader extends URLClassLoader{
method JarLoader (line 19) | public JarLoader(String[] paths) {
method JarLoader (line 23) | public JarLoader(String[] paths, ClassLoader parent) {
method getURLs (line 27) | private static URL[] getURLs(String[] paths) {
method collectDirs (line 45) | private static void collectDirs(String path, List<String> collector) {
method doGetURLs (line 65) | private static List<URL> doGetURLs(final String path) {
FILE: core/src/main/java/com/alibaba/datax/core/util/container/LoadUtil.java
class LoadUtil (line 24) | public class LoadUtil {
method LoadUtil (line 27) | private LoadUtil() {
type ContainerType (line 30) | private enum ContainerType {
method ContainerType (line 34) | private ContainerType(String type) {
method value (line 38) | public String value() {
method bind (line 59) | public static void bind(Configuration pluginConfigs) {
method generatePluginKey (line 63) | private static String generatePluginKey(PluginType pluginType,
method getPluginConf (line 69) | private static Configuration getPluginConf(PluginType pluginType,
method loadJobPlugin (line 91) | public static AbstractJobPlugin loadJobPlugin(PluginType pluginType,
method loadTaskPlugin (line 116) | public static AbstractTaskPlugin loadTaskPlugin(PluginType pluginType,
method loadPluginRunner (line 140) | public static AbstractRunner loadPluginRunner(PluginType pluginType, S...
method loadPluginClass (line 165) | @SuppressWarnings("unchecked")
method getJarLoader (line 180) | public static synchronized JarLoader getJarLoader(PluginType pluginType,
FILE: core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/EnumStrVal.java
type EnumStrVal (line 3) | public interface EnumStrVal {
method value (line 4) | public String value();
FILE: core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/EnumVal.java
type EnumVal (line 3) | public interface EnumVal {
method value (line 4) | public int value();
FILE: core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/ExecuteMode.java
type ExecuteMode (line 3) | public enum ExecuteMode implements EnumStrVal {
method ExecuteMode (line 11) | ExecuteMode(String value) {
method value (line 15) | @Override
method getValue (line 20) | public String getValue() {
method isLocal (line 24) | public static boolean isLocal(String mode) {
method isDistribute (line 28) | public static boolean isDistribute(String mode) {
method toExecuteMode (line 32) | public static ExecuteMode toExecuteMode(String modeName) {
method equalsIgnoreCase (line 41) | private static boolean equalsIgnoreCase(String str1, String str2) {
method toString (line 45) | @Override
FILE: core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/State.java
type State (line 3) | public enum State implements EnumVal {
method State (line 17) | State(int value) {
method value (line 21) | @Override
method isFinished (line 27) | public boolean isFinished() {
method isRunning (line 31) | public boolean isRunning() {
FILE: databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/DatabendWriter.java
class DatabendWriter (line 20) | public class DatabendWriter extends Writer {
class Job (line 23) | public static class Job
method init (line 29) | @Override
method preCheck (line 38) | @Override
method prepare (line 44) | @Override
method split (line 49) | @Override
method post (line 54) | @Override
method destroy (line 59) | @Override
class Task (line 66) | public static class Task extends Writer.Task {
method init (line 73) | @Override
method destroy (line 220) | @Override
method prepare (line 225) | @Override
method post (line 230) | @Override
method startWrite (line 235) | @Override
FILE: databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/DatabendWriterErrorCode.java
type DatabendWriterErrorCode (line 6) | public enum DatabendWriterErrorCode implements ErrorCode {
method DatabendWriterErrorCode (line 14) | private DatabendWriterErrorCode(String code, String description) {
method getCode (line 19) | @Override
method getDescription (line 24) | @Override
method toString (line 29) | @Override
FILE: databendwriter/src/main/java/com/alibaba/datax/plugin/writer/databendwriter/util/DatabendWriterUtil.java
class DatabendWriterUtil (line 17) | public final class DatabendWriterUtil {
method DatabendWriterUtil (line 20) | private DatabendWriterUtil() {
method dealWriteMode (line 23) | public static void dealWriteMode(Configuration originalConfig) throws ...
method onConFlictDoString (line 67) | public static String onConFlictDoString(List<String> conflictColumns) {
FILE: datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java
class Constant (line 3) | public class Constant {
FILE: datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubClientHelper.java
class DatahubClientHelper (line 14) | public class DatahubClientHelper {
method getDatahubClient (line 15) | public static DatahubClient getDatahubClient(Configuration jobConfig) {
FILE: datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReader.java
class DatahubReader (line 24) | public class DatahubReader extends Reader {
class Job (line 25) | public static class Job extends Reader.Job {
method init (line 35) | @Override
method validateParameter (line 44) | private void validateParameter(Configuration conf){
method prepare (line 97) | @Override
method destroy (line 118) | @Override
method split (line 122) | @Override
class Task (line 149) | public static class Task extends Reader.Task {
method init (line 169) | @Override
method destroy (line 201) | @Override
method startRead (line 205) | @Override
FILE: datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderErrorCode.java
type DatahubReaderErrorCode (line 5) | public enum DatahubReaderErrorCode implements ErrorCode {
method DatahubReaderErrorCode (line 15) | private DatahubReaderErrorCode(String code, String description) {
method getCode (line 20) | @Override
method getDescription (line 25) | @Override
method toString (line 30) | @Override
FILE: datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubReaderUtils.java
class DatahubReaderUtils (line 19) | public class DatahubReaderUtils {
method getUnixTimeFromDateTime (line 21) | public static long getUnixTimeFromDateTime(String dateTime) throws Par...
method getShardsWithRetry (line 32) | public static List<ShardEntry> getShardsWithRetry(final DatahubClient ...
method getCursorWithRetry (line 51) | public static String getCursorWithRetry(final DatahubClient datahubCli...
method getLatestCursorWithRetry (line 79) | public static String getLatestCursorWithRetry(final DatahubClient data...
method getDatahubSchemaWithRetry (line 98) | public static RecordSchema getDatahubSchemaWithRetry(final DatahubClie...
method getRecordsResultWithRetry (line 116) | public static GetRecordsResult getRecordsResultWithRetry(final Datahub...
method getColumnFromField (line 136) | public static Column getColumnFromField(RecordEntry record, Field fiel...
FILE: datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubWriterErrorCode.java
type DatahubWriterErrorCode (line 6) | public enum DatahubWriterErrorCode implements ErrorCode {
method DatahubWriterErrorCode (line 17) | private DatahubWriterErrorCode(String code, String description) {
method getCode (line 22) | @Override
method getDescription (line 27) | @Override
method toString (line 32) | @Override
FILE: datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Key.java
class Key (line 3) | public final class Key {
FILE: datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubClientHelper.java
class DatahubClientHelper (line 15) | public class DatahubClientHelper {
method getDatahubClient (line 16) | public static DatahubClient getDatahubClient(Configuration jobConfig) {
FILE: datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriter.java
class DatahubWriter (line 35) | public class DatahubWriter extends Writer {
class Job (line 50) | public static class Job extends Writer.Job {
method init (line 56) | @Override
method prepare (line 66) | @Override
method split (line 88) | @Override
method post (line 97) | @Override
method destroy (line 100) | @Override
class Task (line 105) | public static class Task extends Writer.Task {
method init (line 131) | @Override
method prepare (line 154) | @Override
method startWrite (line 210) | @Override
method post (line 248) | @Override
method destroy (line 251) | @Override
method commit (line 254) | private void commit(List<RecordEntry> records) throws InterruptedExc...
method convertRecord (line 295) | private RecordEntry convertRecord(Record dxRecord, String shardId) {
FILE: datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/DatahubWriterErrorCode.java
type DatahubWriterErrorCode (line 6) | public enum DatahubWriterErrorCode implements ErrorCode {
method DatahubWriterErrorCode (line 17) | private DatahubWriterErrorCode(String code, String description) {
method getCode (line 22) | @Override
method getDescription (line 27) | @Override
method toString (line 32) | @Override
FILE: datahubwriter/src/main/java/com/alibaba/datax/plugin/writer/datahubwriter/Key.java
class Key (line 3) | public final class Key {
FILE: datax-example/datax-example-core/src/main/java/com/alibaba/datax/example/ExampleContainer.java
class ExampleContainer (line 13) | public class ExampleContainer {
method start (line 19) | public static void start(String jobPath) {
FILE: datax-example/datax-example-core/src/main/java/com/alibaba/datax/example/Main.java
class Main (line 9) | public class Main {
method main (line 16) | public static void main(String[] args) {
FILE: datax-example/datax-example-core/src/main/java/com/alibaba/datax/example/util/ExampleConfigParser.java
class ExampleConfigParser (line 18) | public class ExampleConfigParser {
method parse (line 27) | public static Configuration parse(final String jobPath) {
method parsePluginsConfig (line 43) | private static Configuration parsePluginsConfig(Map<String, String> pl...
method runtimeBasePackages (line 79) | private static File[] runtimeBasePackages() {
method scanPluginByPackage (line 106) | private static void scanPluginByPackage(File packageFile,
method parseOnePlugin (line 132) | private static Configuration parseOnePlugin(String packagePath,
method coreConfig (line 145) | private static Configuration coreConfig() {
FILE: datax-example/datax-example-core/src/main/java/com/alibaba/datax/example/util/PathUtil.java
class PathUtil (line 14) | public class PathUtil {
method getAbsolutePathFromClassPath (line 15) | public static String getAbsolutePathFromClassPath(String path) {
FILE: datax-example/datax-example-core/src/test/java/com/alibaba/datax/example/util/PathUtilTest.java
class PathUtilTest (line 11) | public class PathUtilTest {
method testParseClassPathFile (line 13) | @Test
FILE: datax-example/datax-example-neo4j/src/test/java/com/alibaba/datax/example/neo4j/StreamReader2Neo4jWriterTest.java
class StreamReader2Neo4jWriterTest (line 31) | public class StreamReader2Neo4jWriterTest {
method init (line 50) | @Before
method streamReader2Neo4j (line 82) | @Test
method deleteHistoryIfExist (line 96) | private void deleteHistoryIfExist() {
method verifyWriteResult (line 104) | private void verifyWriteResult() {
method destroy (line 118) | @After
method initConnection (line 131) | private void initConnection() {
FILE: datax-example/datax-example-streamreader/src/test/java/com/alibaba/datax/example/streamreader/StreamReader2StreamWriterTest.java
class StreamReader2StreamWriterTest (line 12) | public class StreamReader2StreamWriterTest {
method testStreamReader2StreamWriter (line 13) | @Test
FILE: dorisreader/src/main/java/com/alibaba/datax/plugin/reader/dorisreader/DorisReader.java
class DorisReader (line 14) | public class DorisReader extends Reader {
class Job (line 18) | public static class Job extends Reader.Job {
method init (line 25) | @Override
method preCheck (line 36) | @Override
method split (line 43) | @Override
method post (line 49) | @Override
method destroy (line 54) | @Override
class Task (line 61) | public static class Task extends Reader.Task {
method init (line 66) | @Override
method startRead (line 74) | @Override
method post (line 82) | @Override
method destroy (line 87) | @Override
FILE: dorisreader/src/main/java/com/alibaba/datax/plugin/reader/dorisreader/DorisReaderErrorCode.java
type DorisReaderErrorCode (line 5) | public enum DorisReaderErrorCode implements ErrorCode {
method DorisReaderErrorCode (line 11) | private DorisReaderErrorCode(String code, String description) {
method getCode (line 16) | @Override
method getDescription (line 21) | @Override
method toString (line 26) | @Override
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DelimiterParser.java
class DelimiterParser (line 7) | public class DelimiterParser {
method parse (line 11) | public static String parse(String sp, String dSp) throws RuntimeExcept...
method hexStrToBytes (line 39) | private static byte[] hexStrToBytes(String hexStr) {
method charToByte (line 51) | private static byte charToByte(char c) {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisBaseCodec.java
class DorisBaseCodec (line 5) | public class DorisBaseCodec {
method convertionField (line 6) | protected String convertionField( Column col) {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodec.java
type DorisCodec (line 7) | public interface DorisCodec extends Serializable {
method codec (line 9) | String codec( Record row);
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCodecFactory.java
class DorisCodecFactory (line 5) | public class DorisCodecFactory {
method DorisCodecFactory (line 6) | public DorisCodecFactory (){
method createCodec (line 9) | public static DorisCodec createCodec( Keys writerOptions) {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisCsvCodec.java
class DorisCsvCodec (line 5) | public class DorisCsvCodec extends DorisBaseCodec implements DorisCodec {
method DorisCsvCodec (line 11) | public DorisCsvCodec ( String sp) {
method codec (line 15) | @Override
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisJsonCodec.java
class DorisJsonCodec (line 10) | public class DorisJsonCodec extends DorisBaseCodec implements DorisCodec {
method DorisJsonCodec (line 16) | public DorisJsonCodec ( List<String> fieldNames) {
method codec (line 20) | @Override
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisStreamLoadObserver.java
class DorisStreamLoadObserver (line 32) | public class DorisStreamLoadObserver {
method DorisStreamLoadObserver (line 47) | public DorisStreamLoadObserver ( Keys options){
method streamLoad (line 51) | public void streamLoad(WriterTuple data) throws Exception {
method checkStreamLoadState (line 81) | private void checkStreamLoadState(String host, String label) throws IO...
method addRows (line 126) | private byte[] addRows(List<byte[]> rows, int totalBytes) {
method put (line 155) | private Map<String, Object> put(String loadUrl, String label, byte[] d...
method getBasicAuthHeader (line 192) | private String getBasicAuthHeader(String username, String password) {
method getHttpEntity (line 198) | private HttpEntity getHttpEntity(CloseableHttpResponse resp) {
method getLoadHost (line 212) | private String getLoadHost() {
method checkConnection (line 222) | private boolean checkConnection(String host) {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisUtil.java
class DorisUtil (line 22) | public class DorisUtil {
method DorisUtil (line 25) | private DorisUtil() {}
method getDorisTableColumns (line 27) | public static List<String> getDorisTableColumns( Connection conn, Stri...
method renderPreOrPostSqls (line 45) | public static List<String> renderPreOrPostSqls(List<String> preOrPostS...
method executeSqls (line 58) | public static void executeSqls(Connection conn, List<String> sqls) {
method preCheckPrePareSQL (line 74) | public static void preCheckPrePareSQL( Keys options) {
method preCheckPostSQL (line 90) | public static void preCheckPostSQL( Keys options) {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriter.java
class DorisWriter (line 38) | public class DorisWriter extends Writer {
class Job (line 40) | public static class Job extends Writer.Job {
method init (line 46) | @Override
method preCheck (line 53) | @Override
method prepare (line 60) | @Override
method split (line 74) | @Override
method post (line 83) | @Override
method destroy (line 97) | @Override
class Task (line 103) | public static class Task extends Writer.Task {
method init (line 108) | @Override
method prepare (line 120) | @Override
method startWrite (line 124) | public void startWrite(RecordReceiver recordReceiver) {
method post (line 147) | @Override
method destroy (line 156) | @Override
method supportFailOver (line 159) | @Override
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterExcetion.java
class DorisWriterExcetion (line 6) | public class DorisWriterExcetion extends IOException {
method DorisWriterExcetion (line 11) | public DorisWriterExcetion ( String message, Map<String, Object> respo...
method DorisWriterExcetion (line 16) | public DorisWriterExcetion ( String message, Map<String, Object> respo...
method getFailedResponse (line 22) | public Map<String, Object> getFailedResponse() {
method needReCreateLabel (line 26) | public boolean needReCreateLabel() {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterManager.java
class DorisWriterManager (line 19) | public class DorisWriterManager {
method DorisWriterManager (line 34) | public DorisWriterManager( Keys options) {
method startScheduler (line 42) | public void startScheduler() {
method stopScheduler (line 63) | public void stopScheduler() {
method writeRecord (line 70) | public final synchronized void writeRecord(String record) throws IOExc...
method flush (line 87) | public synchronized void flush(String label, boolean waitUtilDone) thr...
method close (line 105) | public synchronized void close() {
method createBatchLabel (line 119) | public String createBatchLabel() {
method startAsyncFlushing (line 128) | private void startAsyncFlushing() {
method waitAsyncFlushingDone (line 145) | private void waitAsyncFlushingDone() throws InterruptedException {
method asyncFlush (line 153) | private void asyncFlush() throws Exception {
method checkFlushException (line 187) | private void checkFlushException() {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/Keys.java
class Keys (line 12) | public class Keys implements Serializable {
type StreamLoadFormat (line 20) | public enum StreamLoadFormat {
method Keys (line 50) | public Keys ( Configuration options) {
method doPretreatment (line 58) | public void doPretreatment() {
method getJdbcUrl (line 63) | public String getJdbcUrl() {
method getDatabase (line 67) | public String getDatabase() {
method getTable (line 71) | public String getTable() {
method getUsername (line 75) | public String getUsername() {
method getPassword (line 79) | public String getPassword() {
method getLabelPrefix (line 83) | public String getLabelPrefix() {
method getLoadUrlList (line 88) | public List<String> getLoadUrlList() {
method getColumns (line 92) | public List<String> getColumns() {
method isWildcardColumn (line 99) | public boolean isWildcardColumn() {
method setInfoCchemaColumns (line 103) | public void setInfoCchemaColumns(List<String> cols) {
method getPreSqlList (line 107) | public List<String> getPreSqlList() {
method getPostSqlList (line 111) | public List<String> getPostSqlList() {
method getLoadProps (line 115) | public Map<String, Object> getLoadProps() {
method getMaxRetries (line 119) | public int getMaxRetries() {
method getBatchRows (line 123) | public int getBatchRows() {
method getBatchSize (line 128) | public long getBatchSize() {
method getFlushInterval (line 133) | public long getFlushInterval() {
method getFlushQueueLength (line 138) | public int getFlushQueueLength() {
method getStreamLoadFormat (line 143) | public StreamLoadFormat getStreamLoadFormat() {
method validateStreamLoadUrl (line 155) | private void validateStreamLoadUrl() {
method validateRequired (line 165) | private void validateRequired() {
FILE: doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/WriterTuple.java
class WriterTuple (line 5) | public class WriterTuple {
method WriterTuple (line 10) | public WriterTuple ( String label, Long bytes, List<byte[]> rows){
method getLabel (line 16) | public String getLabel() { return label; }
method setLabel (line 17) | public void setLabel(String label) { this.label = label; }
method getBytes (line 18) | public Long getBytes() { return bytes; }
method getRows (line 19) | public List<byte[]> getRows() { return rows; }
FILE: drdsreader/src/main/java/com/alibaba/datax/plugin/reader/drdsreader/DrdsReader.java
class DrdsReader (line 18) | public class DrdsReader extends Reader {
class Job (line 23) | public static class Job extends Reader.Job {
method init (line 28) | @Override
method split (line 41) | @Override
method post (line 47) | @Override
method destroy (line 52) | @Override
method validateConfiguration (line 57) | private void validateConfiguration() {
class Task (line 116) | public static class Task extends Reader.Task {
method init (line 121) | @Override
method startRead (line 130) | @Override
method post (line 138) | @Override
method destroy (line 143) | @Override
FILE: drdsreader/src/main/java/com/alibaba/datax/plugin/reader/drdsreader/DrdsReaderErrorCode.java
type DrdsReaderErrorCode (line 5) | public enum DrdsReaderErrorCode implements ErrorCode {
method DrdsReaderErrorCode (line 11) | private DrdsReaderErrorCode(String code, String description) {
method getCode (line 16) | @Override
method getDescription (line 21) | @Override
method toString (line 26) | @Override
FILE: drdsreader/src/main/java/com/alibaba/datax/plugin/reader/drdsreader/DrdsReaderSplitUtil.java
class DrdsReaderSplitUtil (line 18) | public class DrdsReaderSplitUtil {
method doSplit (line 23) | public static List<Configuration> doSplit(Configuration originalSliceC...
method doDrdsReaderSplit (line 45) | private static List<Configuration> doDrdsReaderSplit(Configuration ori...
method getTopology (line 87) | private static Map<String, List<String>> getTopology(Configuration con...
FILE: drdswriter/src/main/java/com/alibaba/datax/plugin/writer/drdswriter/DrdsWriter.java
class DrdsWriter (line 15) | public class DrdsWriter extends Writer {
class Job (line 18) | public static class Job extends Writer.Job {
method init (line 24) | @Override
method prepare (line 41) | @Override
method split (line 46) | @Override
method post (line 52) | @Override
method destroy (line 57) | @Override
class Task (line 64) | public static class Task extends Writer.Task {
method init (line 68) | @Override
method prepare (line 75) | @Override
method startWrite (line 81) | public void startWrite(RecordReceiver recordReceiver) {
method post (line 86) | @Override
method destroy (line 91) | @Override
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchClient.java
class ElasticSearchClient (line 43) | public class ElasticSearchClient {
method getClient (line 49) | public JestClient getClient() {
method ElasticSearchClient (line 53) | public ElasticSearchClient(Configuration conf) {
method indicesExists (line 86) | public boolean indicesExists(String indexName) throws Exception {
method deleteIndex (line 107) | public boolean deleteIndex(String indexName) throws Exception {
method isGreaterOrEqualThan7 (line 123) | public boolean isGreaterOrEqualThan7() throws Exception {
method getIndexSettings (line 139) | public String getIndexSettings(String indexName) {
method createIndexIfNotExists (line 154) | public boolean createIndexIfNotExists(String indexName, String typeName,
method execute (line 202) | public <T extends JestResult> T execute(Action<T> clientRequest) throw...
method getStatus (line 210) | public Integer getStatus(JestResult rst) {
method isBulkResult (line 218) | public boolean isBulkResult(JestResult rst) {
method alias (line 224) | public boolean alias(String indexname, String aliasname, boolean needC...
method getIndexMapping (line 259) | public String getIndexMapping(String indexName) {
method getMappingForIndexType (line 274) | public String getMappingForIndexType(String indexName, String typeName) {
method bulkInsert (line 287) | public JestResult bulkInsert(Bulk.Builder bulk) throws Exception {
method closeJestClient (line 303) | public void closeJestClient() {
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchColumn.java
class ElasticSearchColumn (line 8) | public class ElasticSearchColumn {
method getCombineFieldsValueSeparator (line 43) | public String getCombineFieldsValueSeparator() {
method setCombineFieldsValueSeparator (line 47) | public void setCombineFieldsValueSeparator(String combineFieldsValueSe...
method getCombineFields (line 51) | public List<String> getCombineFields() {
method setCombineFields (line 55) | public void setCombineFields(List<String> combineFields) {
method setName (line 59) | public void setName(String name) {
method setType (line 63) | public void setType(String type) {
method setTimeZone (line 67) | public void setTimeZone(String timezone) {
method setFormat (line 71) | public void setFormat(String format) {
method getName (line 75) | public String getName() {
method getType (line 79) | public String getType() {
method isOrigin (line 83) | public boolean isOrigin() { return origin; }
method setOrigin (line 85) | public void setOrigin(boolean origin) { this.origin = origin; }
method getTimezone (line 87) | public String getTimezone() {
method getFormat (line 91) | public String getFormat() {
method setTimezone (line 95) | public void setTimezone(String timezone) {
method isArray (line 99) | public boolean isArray() {
method setArray (line 103) | public void setArray(boolean array) {
method isJsonArray (line 107) | public boolean isJsonArray() {return jsonArray;}
method setJsonArray (line 109) | public void setJsonArray(boolean jsonArray) {this.jsonArray = jsonArray;}
method getDstFormat (line 111) | public String getDstFormat() {
method setDstFormat (line 115) | public void setDstFormat(String dstFormat) {
method isDstArray (line 119) | public boolean isDstArray() {
method setDstArray (line 123) | public void setDstArray(boolean dstArray) {
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchFieldType.java
type ElasticSearchFieldType (line 6) | public enum ElasticSearchFieldType {
method getESFieldType (line 37) | public static ElasticSearchFieldType getESFieldType(String type) {
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriter.java
class ElasticSearchWriter (line 33) | public class ElasticSearchWriter extends Writer {
class Job (line 36) | public static class Job extends Writer.Job {
method setSettings (line 45) | private void setSettings(String settings) {
method init (line 49) | @Override
method getIncludeSettings (line 57) | public List<String> getIncludeSettings() {
method convertSettings (line 66) | private String convertSettings(String originSettings) {
method prepare (line 94) | @Override
method hasID (line 179) | private boolean hasID() {
method hasPrimaryKeyInfo (line 194) | private boolean hasPrimaryKeyInfo() {
method genMappings (line 204) | private String genMappings(String dstDynamic, String typeName, boole...
method split (line 379) | @Override
method post (line 388) | @Override
method destroy (line 402) | @Override
class Task (line 408) | public static class Task extends Writer.Task {
method init (line 443) | @Override
method handleMetaKeys (line 485) | private void handleMetaKeys() {
method parseDeleteCondition (line 532) | private void parseDeleteCondition(Configuration conf) {
method prepare (line 545) | @Override
method isDeleteRecord (line 565) | private boolean isDeleteRecord(Record record) {
method meetAllCondition (line 586) | private boolean meetAllCondition(Map<String, Object> kv, Map<String,...
method checkOneCondition (line 595) | private boolean checkOneCondition(Map<String, Object> kv, Map.Entry<...
method startWrite (line 611) | @Override
method getDateStr (line 644) | private String getDateStr(ElasticSearchColumn esColumn, Column colum...
method doBatchInsert (line 673) | private void doBatchInsert(final List<Record> writerBuffer) {
method getRecordColumnIndex (line 1060) | private int getRecordColumnIndex(Record record, String columnName) {
method processIDCombineFields (line 1090) | private String processIDCombineFields(Record record, ElasticSearchCo...
method post (line 1103) | @Override
method destroy (line 1107) | @Override
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ElasticSearchWriterErrorCode.java
type ElasticSearchWriterErrorCode (line 5) | public enum ElasticSearchWriterErrorCode implements ErrorCode {
method ElasticSearchWriterErrorCode (line 21) | ElasticSearchWriterErrorCode(String code, String description) {
method getCode (line 26) | @Override
method getDescription (line 31) | @Override
method toString (line 36) | @Override
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonPathUtil.java
class JsonPathUtil (line 7) | public class JsonPathUtil {
method getJsonObject (line 9) | public static JSONObject getJsonObject(List<String> paths, JSONObject ...
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/JsonUtil.java
class JsonUtil (line 11) | public class JsonUtil {
method mergeJsonStr (line 20) | public static String mergeJsonStr(String source, String target) throws...
method deepMerge (line 37) | private static JSONObject deepMerge(JSONObject source, JSONObject targ...
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/Key.java
class Key (line 13) | public final class Key {
type ActionType (line 19) | public static enum ActionType {
method getActionType (line 27) | public static ActionType getActionType(Configuration conf) {
method getEndpoint (line 43) | public static String getEndpoint(Configuration conf) {
method getUsername (line 47) | public static String getUsername(Configuration conf) {
method getPassword (line 51) | public static String getPassword(Configuration conf) {
method getBatchSize (line 55) | public static int getBatchSize(Configuration conf) {
method getTrySize (line 59) | public static int getTrySize(Configuration conf) {
method getTryInterval (line 63) | public static long getTryInterval(Configuration conf) {
method getTimeout (line 67) | public static int getTimeout(Configuration conf) {
method isTruncate (line 71) | public static boolean isTruncate(Configuration conf) {
method isDiscovery (line 75) | public static boolean isDiscovery(Configuration conf) {
method isCompression (line 79) | public static boolean isCompression(Configuration conf) {
method isMultiThread (line 83) | public static boolean isMultiThread(Configuration conf) {
method getIndexName (line 87) | public static String getIndexName(Configuration conf) {
method getDeleteBy (line 91) | public static String getDeleteBy(Configuration conf) {
method getTypeName (line 99) | public static String getTypeName(Configuration conf) {
method isIgnoreWriteError (line 108) | public static boolean isIgnoreWriteError(Configuration conf) {
method isIgnoreParseError (line 112) | public static boolean isIgnoreParseError(Configuration conf) {
method isHighSpeedMode (line 117) | public static boolean isHighSpeedMode(Configuration conf) {
method getAlias (line 124) | public static String getAlias(Configuration conf) {
method isNeedCleanAlias (line 128) | public static boolean isNeedCleanAlias(Configuration conf) {
method getSettings (line 136) | public static Map<String, Object> getSettings(Configuration conf) {
method getSplitter (line 140) | public static String getSplitter(Configuration conf) {
method getDynamic (line 144) | public static boolean getDynamic(Configuration conf) {
method getDstDynamic (line 148) | public static String getDstDynamic(Configuration conf) {
method getDiscoveryFilter (line 152) | public static String getDiscoveryFilter(Configuration conf){
method getVersioning (line 156) | public static Boolean getVersioning(Configuration conf) {
method getUnifiedVersion (line 160) | public static Long getUnifiedVersion(Configuration conf) {
method getUrlParams (line 164) | public static Map<String, Object> getUrlParams(Configuration conf) {
method getESVersion (line 168) | public static Integer getESVersion(Configuration conf) {
method getMasterTimeout (line 172) | public static String getMasterTimeout(Configuration conf) {
method isEnableNullUpdate (line 176) | public static boolean isEnableNullUpdate(Configuration conf) {
method getFieldDelimiter (line 180) | public static String getFieldDelimiter(Configuration conf) {
method getPrimaryKeyInfo (line 184) | public static PrimaryKeyInfo getPrimaryKeyInfo(Configuration conf) {
method getEsPartitionColumn (line 193) | public static List<PartitionColumn> getEsPartitionColumn(Configuration...
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/NoReRunException.java
class NoReRunException (line 6) | public class NoReRunException extends DataXException {
method NoReRunException (line 7) | public NoReRunException(String errorMessage) {
method NoReRunException (line 11) | public NoReRunException(ErrorCode errorCode, String errorMessage) {
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PartitionColumn.java
class PartitionColumn (line 3) | public class PartitionColumn {
method getName (line 11) | public String getName() {
method getMetaType (line 15) | public String getMetaType() {
method getComment (line 19) | public String getComment() {
method getType (line 23) | public String getType() {
method setName (line 27) | public void setName(String name) {
method setMetaType (line 31) | public void setMetaType(String metaType) {
method setComment (line 35) | public void setComment(String comment) {
method setType (line 39) | public void setType(String type) {
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/PrimaryKeyInfo.java
class PrimaryKeyInfo (line 5) | public class PrimaryKeyInfo {
method getType (line 24) | public String getType() {
method getFieldDelimiter (line 28) | public String getFieldDelimiter() {
method getColumn (line 32) | public List<String> getColumn() {
method setType (line 36) | public void setType(String type) {
method setFieldDelimiter (line 40) | public void setFieldDelimiter(String fieldDelimiter) {
method setColumn (line 44) | public void setColumn(List<String> column) {
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfo.java
class ClusterInfo (line 7) | public class ClusterInfo extends AbstractAction<ClusterInfoResult> {
method buildURI (line 8) | @Override
method getRestMethodName (line 13) | @Override
method createNewElasticSearchResult (line 18) | @Override
class Builder (line 23) | public static class Builder extends AbstractAction.Builder<ClusterInfo...
method Builder (line 25) | public Builder() {
method build (line 30) | @Override
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/ClusterInfoResult.java
class ClusterInfoResult (line 9) | public class ClusterInfoResult extends JestResult {
method ClusterInfoResult (line 15) | public ClusterInfoResult(Gson gson) {
method ClusterInfoResult (line 19) | public ClusterInfoResult(JestResult source) {
method isGreaterOrEqualThan7 (line 29) | public Boolean isGreaterOrEqualThan7() throws Exception {
FILE: elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/jest/PutMapping7.java
class PutMapping7 (line 6) | public class PutMapping7 extends GenericResultAbstractAction {
method PutMapping7 (line 7) | protected PutMapping7(PutMapping7.Builder builder) {
method buildURI (line 14) | @Override
method getRestMethodName (line 19) | @Override
class Builder (line 24) | public static class Builder extends GenericResultAbstractAction.Builde...
method Builder (line 28) | public Builder(String index, Object source) {
method build (line 33) | @Override
FILE: ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/Constant.java
class Constant (line 4) | public class Constant {
FILE: ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/FtpHelper.java
class FtpHelper (line 7) | public abstract class FtpHelper {
method loginFtpServer (line 21) | public abstract void loginFtpServer(String host, String username, Stri...
method logoutFtpServer (line 31) | public abstract void logoutFtpServer();
method isDirExist (line 41) | public abstract boolean isDirExist(String directoryPath);
method isFileExist (line 51) | public abstract boolean isFileExist(String filePath);
method isSymbolicLink (line 61) | public abstract boolean isSymbolicLink(String filePath);
method getListFiles (line 73) | public abstract HashSet<String> getListFiles(String directoryPath, int...
method getInputStream (line 84) | public abstract InputStream getInputStream(String filePath);
method getAllFiles (line 97) | public HashSet<String> getAllFiles(List<String> srcPaths, int parentLe...
FILE: ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/FtpReader.java
class FtpReader (line 17) | public class FtpReader extends Reader {
class Job (line 18) | public static class Job extends Reader.Job {
method init (line 39) | @Override
method validateParameter (line 60) | private void validateParameter() {
method prepare (line 105) | @Override
method post (line 114) | @Override
method destroy (line 118) | @Override
method split (line 131) | @Override
method splitSourceFiles (line 154) | private <T> List<List<T>> splitSourceFiles(final List<T> sourceList,...
class Task (line 171) | public static class Task extends Reader.Task {
method init (line 187) | @Override
method prepare (line 213) | @Override
method post (line 218) | @Override
method destroy (line 223) | @Override
method startRead (line 235) | @Override
FILE: ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/FtpReaderErrorCode.java
type FtpReaderErrorCode (line 8) | public enum FtpReaderErrorCode implements ErrorCode {
method FtpReaderErrorCode (line 32) | private FtpReaderErrorCode(String code, String description) {
method getCode (line 37) | @Override
method getDescription (line 42) | @Override
method toString (line 47) | @Override
FILE: ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/Key.java
class Key (line 3) | public class Key {
FILE: ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/SftpHelper.java
class SftpHelper (line 22) | public class SftpHelper extends FtpHelper {
method loginFtpServer (line 27) | @Override
method logoutFtpServer (line 87) | @Override
method isDirExist (line 97) | @Override
method isFileExist (line 114) | @Override
method isSymbolicLink (line 136) | @Override
method getListFiles (line 155) | @Override
method getInputStream (line 237) | @Override
FILE: ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/StandardFtpHelper.java
class StandardFtpHelper (line 20) | public class StandardFtpHelper extends FtpHelper {
method loginFtpServer (line 24) | @Override
method logoutFtpServer (line 72) | @Override
method isDirExist (line 97) | @Override
method isFileExist (line 108) | @Override
method isSymbolicLink (line 124) | @Override
method getListFiles (line 141) | @Override
method getInputStream (line 218) | @Override
FILE: ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/FtpWriter.java
class FtpWriter (line 25) | public class FtpWriter extends Writer {
class Job (line 26) | public static class Job extends Writer.Job {
method init (line 41) | @Override
method validateParameter (line 66) | private void validateParameter() {
method prepare (line 108) | @Override
method post (line 172) | @Override
method destroy (line 177) | @Override
method split (line 189) | @Override
class Task (line 197) | public static class Task extends Writer.Task {
method init (line 215) | @Override
method prepare (line 256) | @Override
method startWrite (line 261) | @Override
method post (line 284) | @Override
method destroy (line 289) | @Override
FILE: ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/FtpWriterErrorCode.java
type FtpWriterErrorCode (line 5) | public enum FtpWriterErrorCode implements ErrorCode {
method FtpWriterErrorCode (line 33) | private FtpWriterErrorCode(String code, String description) {
method getCode (line 38) | @Override
method getDescription (line 43) | @Override
method toString (line 48) | @Override
FILE: ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/Key.java
class Key (line 3) | public class Key {
FILE: ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/Constant.java
class Constant (line 4) | public class Constant {
FILE: ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/IFtpHelper.java
type IFtpHelper (line 6) | public interface IFtpHelper {
method loginFtpServer (line 9) | public void loginFtpServer(String host, String username, String passwo...
method logoutFtpServer (line 11) | public void logoutFtpServer();
method mkdir (line 16) | public void mkdir(String directoryPath);
method mkDirRecursive (line 21) | public void mkDirRecursive(String directoryPath);
method getOutputStream (line 23) | public OutputStream getOutputStream(String filePath);
method getRemoteFileContent (line 25) | public String getRemoteFileContent(String filePath);
method getAllFilesInDir (line 27) | public Set<String> getAllFilesInDir(String dir, String prefixFileName);
method deleteFiles (line 32) | public void deleteFiles(Set<String> filesToDelete);
method completePendingCommand (line 34) | public void completePendingCommand();
FILE: ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/SftpHelperImpl.java
class SftpHelperImpl (line 27) | public class SftpHelperImpl implements IFtpHelper {
method loginFtpServer (line 34) | @Override
method logoutFtpServer (line 92) | @Override
method mkdir (line 104) | @Override
method mkDirRecursive (line 136) | @Override
method mkDirSingleHierarchy (line 175) | public boolean mkDirSingleHierarchy(String directoryPath) throws SftpE...
method getOutputStream (line 194) | @Override
method getRemoteFileContent (line 222) | @Override
method getAllFilesInDir (line 246) | @Override
method deleteFiles (line 273) | @Override
method printWorkingDirectory (line 293) | private void printWorkingDirectory() {
method completePendingCommand (line 303) | @Override
FILE: ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/StandardFtpHelperImpl.java
class StandardFtpHelperImpl (line 24) | public class StandardFtpHelperImpl implements IFtpHelper {
method loginFtpServer (line 29) | @Override
method logoutFtpServer (line 82) | @Override
method mkdir (line 111) | @Override
method mkDirRecursive (line 139) | @Override
method mkDirSingleHierarchy (line 166) | public boolean mkDirSingleHierarchy(String directoryPath) throws IOExc...
method getOutputStream (line 180) | @Override
method getRemoteFileContent (line 209) | @Override
method getAllFilesInDir (line 233) | @Override
method deleteFiles (line 265) | @Override
method printWorkingDirectory (line 293) | private void printWorkingDirectory() {
method completePendingCommand (line 303) | @Override
FILE: gaussdbreader/src/main/java/com/alibaba/datax/plugin/reader/gaussdbreader/Constant.java
class Constant (line 3) | public class Constant {
FILE: gaussdbreader/src/main/java/com/alibaba/datax/plugin/reader/gaussdbreader/GaussDbReader.java
class GaussDbReader (line 13) | public class GaussDbReader extends Reader {
class Job (line 17) | public static class Job extends Reader.Job {
method init (line 22) | @Override
method split (line 37) | @Override
method post (line 42) | @Override
method destroy (line 47) | @Override
class Task (line 54) | public static class Task extends Reader.Task {
method init (line 59) | @Override
method startRead (line 66) | @Override
method post (line 74) | @Override
method destroy (line 79) | @Override
FILE: gaussdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gaussdbwriter/GaussDbWriter.java
class GaussDbWriter (line 14) | public class GaussDbWriter extends Writer {
class Job (line 18) | public static class Job extends Writer.Job {
method init (line 22) | @Override
method prepare (line 37) | @Override
method split (line 42) | @Override
method post (line 47) | @Override
method destroy (line 52) | @Override
class Task (line 59) | public static class Task extends Writer.Task {
method init (line 63) | @Override
method prepare (line 82) | @Override
method startWrite (line 87) | public void startWrite(RecordReceiver recordReceiver) {
method post (line 91) | @Override
method destroy (line 96) | @Override
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/GdbReader.java
class GdbReader (line 22) | public class GdbReader extends Reader {
class Job (line 40) | public static class Job extends Reader.Job {
method init (line 45) | @Override
method prepare (line 64) | @Override
method split (line 78) | @Override
method post (line 105) | @Override
method destroy (line 113) | @Override
class Task (line 128) | public static class Task extends Reader.Task {
method init (line 137) | @Override
method prepare (line 151) | @Override
method startRead (line 159) | @Override
method post (line 213) | @Override
method destroy (line 221) | @Override
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/GdbReaderErrorCode.java
type GdbReaderErrorCode (line 5) | public enum GdbReaderErrorCode implements ErrorCode {
method GdbReaderErrorCode (line 19) | private GdbReaderErrorCode(String code, String description) {
method getCode (line 24) | @Override
method getDescription (line 29) | @Override
method toString (line 34) | @Override
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/Key.java
class Key (line 3) | public final class Key {
type ExportType (line 24) | public enum ExportType {
type ColumnType (line 35) | public enum ColumnType {
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/mapping/DefaultGdbMapper.java
class DefaultGdbMapper (line 26) | public class DefaultGdbMapper {
method getMapper (line 28) | public static BiConsumer<GdbElement, Record> getMapper(MappingRule rul...
method forEdgePropertyValue (line 79) | private static Function<Object, Object> forEdgePropertyValue() {
method forVertexOnePropertyValue (line 93) | private static Function<Object, Object> forVertexOnePropertyValue() {
method forEdgeJsonProperties (line 111) | private static Function<Map<String, Object>, String> forEdgeJsonProper...
method forVertexJsonProperties (line 126) | private static Function<Map<String, Object>, String> forVertexJsonProp...
method forVertexPropertyStr (line 139) | private static Function<List<?>, String> forVertexPropertyStr() {
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/mapping/MappingRule.java
class MappingRule (line 24) | @Data
method addColumn (line 37) | void addColumn(ColumnType columnType, ValueType type, String name) {
method addJsonColumn (line 57) | void addJsonColumn(ColumnType columnType) {
class ColumnMappingRule (line 71) | @Data
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/mapping/MappingRuleFactory.java
class MappingRuleFactory (line 25) | public class MappingRuleFactory {
method getInstance (line 28) | public static MappingRuleFactory getInstance() {
method create (line 32) | public MappingRule create(Configuration config, ExportType exportType) {
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/mapping/ValueType.java
type ValueType (line 25) | public enum ValueType {
method ValueType (line 47) | ValueType(Class<?> type, String name, Function<Object, Column> columnF...
method fromShortName (line 55) | public static ValueType fromShortName(String name) {
method applyObject (line 59) | public Column applyObject(Object value) {
class ValueTypeHolder (line 66) | private static class ValueTypeHolder {
method longColumnMapper (line 69) | private static LongColumn longColumnMapper(Object o) {
method doubleColumnMapper (line 84) | private static DoubleColumn doubleColumnMapper(Object o) {
method boolColumnMapper (line 103) | private static BoolColumn boolColumnMapper(Object o) {
method stringColumnMapper (line 120) | private static StringColumn stringColumnMapper(Object o) {
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/AbstractGdbGraph.java
class AbstractGdbGraph (line 30) | public abstract class AbstractGdbGraph implements GdbGraph {
method AbstractGdbGraph (line 35) | AbstractGdbGraph() {
method AbstractGdbGraph (line 38) | AbstractGdbGraph(Configuration config) {
method runInternal (line 60) | protected List<Result> runInternal(String dsl, Map<String, Object> par...
method runInternalAsync (line 64) | protected ResultSet runInternalAsync(String dsl, Map<String, Object> p...
method warmClient (line 72) | private void warmClient() {
method close (line 82) | @Override
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbElement.java
class GdbElement (line 20) | @Data
method GdbElement (line 31) | public GdbElement() {
method GdbElement (line 34) | public GdbElement(String id, String label) {
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbGraph.java
type GdbGraph (line 20) | public interface GdbGraph extends AutoCloseable {
method getLabels (line 27) | Map<String, Long> getLabels();
method fetchIds (line 37) | List<String> fetchIds(String label, String start, long limit);
method fetchElementsAsync (line 48) | ResultSet fetchElementsAsync(String label, String start, String end, L...
method getElement (line 56) | List<GdbElement> getElement(ResultSet results);
method close (line 63) | @Override
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/ScriptGdbGraph.java
class ScriptGdbGraph (line 31) | public class ScriptGdbGraph extends AbstractGdbGraph {
method ScriptGdbGraph (line 54) | public ScriptGdbGraph(ExportType exportType) {
method ScriptGdbGraph (line 59) | public ScriptGdbGraph(Configuration config, ExportType exportType) {
method fetchIds (line 64) | @Override
method fetchElementsAsync (line 86) | @Override
method getElement (line 117) | @Override
method mapNodeToElement (line 149) | private void mapNodeToElement(Object node, GdbElement element) {
method mapPropToElement (line 167) | private void mapPropToElement(Map<String, Object> props, GdbElement el...
method getLabels (line 171) | @Override
FILE: gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/util/ConfigHelper.java
type ConfigHelper (line 27) | public interface ConfigHelper {
method assertConfig (line 28) | static void assertConfig(String key, Supplier<Boolean> f) {
method assertHasContent (line 34) | static void assertHasContent(Configuration config, String key) {
method assertGdbClient (line 38) | static void assertGdbClient(Configuration config) {
method assertLabels (line 46) | static List<String> assertLabels(Configuration config) {
method splitConfig (line 59) | static List<Configuration> splitConfig(Configuration config, List<Stri...
method fromClasspath (line 70) | static Configuration fromClasspath(String name) {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriter.java
class GdbWriter (line 31) | public class GdbWriter extends Writer {
class Job (line 52) | public static class Job extends Writer.Job {
method init (line 57) | @Override
method prepare (line 71) | @Override
method split (line 94) | @Override
method post (line 110) | @Override
method destroy (line 119) | @Override
class Task (line 129) | @Slf4j
method init (line 139) | @Override
method prepare (line 165) | @Override
method startWrite (line 174) | @Override
method wait4Submit (line 209) | private void wait4Submit(final Future<Boolean> future) {
method batchCommitRecords (line 221) | private boolean batchCommitRecords(final List<Tuple2<Record, GdbElem...
method post (line 236) | @Override
method destroy (line 245) | @Override
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriterErrorCode.java
type GdbWriterErrorCode (line 5) | public enum GdbWriterErrorCode implements ErrorCode {
method GdbWriterErrorCode (line 13) | private GdbWriterErrorCode(String code, String description) {
method getCode (line 18) | @Override
method getDescription (line 23) | @Override
method toString (line 28) | @Override
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/Key.java
class Key (line 3) | public final class Key {
type ImportType (line 77) | public static enum ImportType {
type UpdateMode (line 88) | public static enum UpdateMode {
type ColumnType (line 103) | public static enum ColumnType {
type IdTransRule (line 145) | public static enum IdTransRule {
type PropertyType (line 157) | public static enum PropertyType {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbGraphManager.java
class GdbGraphManager (line 17) | public class GdbGraphManager implements AutoCloseable {
method instance (line 22) | public static GdbGraphManager instance() {
method getGraph (line 26) | public GdbGraph getGraph(final Configuration config, final boolean ses...
method close (line 32) | @Override
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/client/GdbWriterConfig.java
class GdbWriterConfig (line 16) | public class GdbWriterConfig {
method GdbWriterConfig (line 28) | private GdbWriterConfig(final Configuration config) {
method of (line 34) | public static GdbWriterConfig of(final Configuration config) {
method validate (line 38) | private void validate() {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/DefaultGdbMapper.java
class DefaultGdbMapper (line 31) | @Slf4j
method DefaultGdbMapper (line 39) | public DefaultGdbMapper() {}
method DefaultGdbMapper (line 41) | public DefaultGdbMapper(final Configuration config) {
method forElement (line 45) | private static BiConsumer<Record, GdbElement> forElement(final Mapping...
method forObjColumn (line 120) | private static Function<Record, Object> forObjColumn(final boolean num...
method forStrColumn (line 131) | private static Function<Record, String> forStrColumn(final boolean num...
method addToProperties (line 166) | private static boolean addToProperties(final GdbElement e, final Strin...
method getMapper (line 196) | @Override
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/GdbMapper.java
type GdbMapper (line 15) | public interface GdbMapper {
method getMapper (line 16) | Function<Record, GdbElement> getMapper(MappingRule rule);
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MapperConfig.java
class MapperConfig (line 18) | public class MapperConfig {
method MapperConfig (line 25) | private MapperConfig() {
method getInstance (line 32) | public static MapperConfig getInstance() {
method updateConfig (line 36) | public void updateConfig(final Configuration config) {
method getMaxIdLength (line 52) | public int getMaxIdLength() {
method getMaxLabelLength (line 56) | public int getMaxLabelLength() {
method getMaxPropKeyLength (line 60) | public int getMaxPropKeyLength() {
method getMaxPropValueLength (line 64) | public int getMaxPropValueLength() {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRule.java
class MappingRule (line 18) | @Data
class PropertyMappingRule (line 36) | @Data
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRuleFactory.java
class MappingRuleFactory (line 26) | @Slf4j
method getInstance (line 32) | public static MappingRuleFactory getInstance() {
method isPattern (line 36) | private static boolean isPattern(final String value, final MappingRule...
method create (line 60) | @Deprecated
method createV2 (line 91) | public MappingRule createV2(final Configuration config) {
method createV2 (line 102) | public MappingRule createV2(final Configuration config, final ImportTy...
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/ValueType.java
type ValueType (line 18) | @Slf4j
method ValueType (line 36) | private ValueType(final Class<?> type, final String name, final Functi...
method fromShortName (line 46) | public static ValueType fromShortName(final String name) {
method type (line 50) | public Class<?> type() {
method shortName (line 54) | public String shortName() {
method applyColumn (line 58) | public Object applyColumn(final Column column) {
method fromStrFunc (line 70) | public Object fromStrFunc(final String str) {
class ValueTypeHolder (line 74) | private static class ValueTypeHolder {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/AbstractGdbGraph.java
class AbstractGdbGraph (line 29) | @Slf4j
method AbstractGdbGraph (line 39) | protected AbstractGdbGraph() {}
method AbstractGdbGraph (line 41) | protected AbstractGdbGraph(final Configuration config, final boolean s...
method initClient (line 45) | protected void initClient(final Configuration config, final boolean se...
method runInternal (line 90) | protected void runInternal(final String dsl, final Map<String, Object>...
method beginTx (line 100) | void beginTx() {
method doCommit (line 109) | void doCommit() {
method doRollback (line 122) | void doRollback() {
method warmClient (line 131) | private void warmClient(final int num) {
method close (line 144) | @Override
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbEdge.java
class GdbEdge (line 15) | @EqualsAndHashCode(callSuper = true)
method getFrom (line 21) | public String getFrom() {
method setFrom (line 25) | public void setFrom(final String from) {
method getTo (line 33) | public String getTo() {
method setTo (line 37) | public void setTo(final String to) {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbElement.java
class GdbElement (line 16) | public class GdbElement {
method getId (line 21) | public String getId() {
method setId (line 25) | public void setId(final String id) {
method getLabel (line 33) | public String getLabel() {
method setLabel (line 37) | public void setLabel(final String label) {
method getProperties (line 45) | public List<GdbProperty> getProperties() {
method addProperty (line 49) | public void addProperty(final String propKey, final Object propValue, ...
method addProperty (line 68) | public void addProperty(final String propKey, final Object propValue) {
method toString (line 72) | @Override
class GdbProperty (line 86) | public static class GdbProperty {
method GdbProperty (line 91) | private GdbProperty(final String key, final Object value, final Prop...
method getCardinality (line 97) | public PropertyType getCardinality() {
method getKey (line 101) | public String getKey() {
method getValue (line 105) | public Object getValue() {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbGraph.java
type GdbGraph (line 16) | public interface GdbGraph extends AutoCloseable {
method add (line 17) | List<Tuple2<Record, Exception>> add(List<Tuple2<Record, GdbElement>> r...
method close (line 19) | @Override
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbVertex.java
class GdbVertex (line 13) | @EqualsAndHashCode(callSuper = true)
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/ScriptGdbGraph.java
class ScriptGdbGraph (line 24) | @Slf4j
method ScriptGdbGraph (line 42) | public ScriptGdbGraph() {
method ScriptGdbGraph (line 46) | public ScriptGdbGraph(final Configuration config, final boolean sessio...
method add (line 60) | @Override
method addInternal (line 80) | private void addInternal(final GdbElement element) {
method addInternal (line 103) | private void addInternal(final GdbElement element, final boolean updat...
method checkSplitDsl (line 142) | private boolean checkSplitDsl(final boolean firstAdd, final int reques...
method buildDsl (line 151) | private Tuple2<String, Map<String, Object>> buildDsl(final GdbElement ...
method setGraphDbElement (line 186) | private void setGraphDbElement(final GdbElement element, final List<Gd...
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/ConfigHelper.java
type ConfigHelper (line 22) | public interface ConfigHelper {
method assertConfig (line 23) | static void assertConfig(final String key, final Supplier<Boolean> f) {
method assertHasContent (line 29) | static void assertHasContent(final Configuration config, final String ...
method getConfig (line 44) | static <T> T getConfig(final Configuration conf, final String key, fin...
method fromClasspath (line 56) | static Configuration fromClasspath(final String name) {
FILE: gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/GdbDuplicateIdException.java
class GdbDuplicateIdException (line 14) | public class GdbDuplicateIdException extends Exception {
method GdbDuplicateIdException (line 15) | public GdbDuplicateIdException(Exception e) {
method GdbDuplicateIdException (line 19) | public GdbDuplicateIdException() {
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/ColumnType.java
type ColumnType (line 11) | public enum ColumnType {
method ColumnType (line 25) | ColumnType(String typeName) {
method getByTypeName (line 29) | public static ColumnType getByTypeName(String typeName) {
method toString (line 44) | @Override
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Constant.java
class Constant (line 3) | public final class Constant {
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Hbase094xHelper.java
class Hbase094xHelper (line 31) | public class Hbase094xHelper {
method getHbaseConf (line 35) | public static org.apache.hadoop.conf.Configuration getHbaseConf(String...
method getTable (line 57) | public static HTable getTable(com.alibaba.datax.common.util.Configurat...
method checkHbaseTable (line 75) | private static void checkHbaseTable(HBaseAdmin admin, HTable htable) t...
method closeAdmin (line 94) | public static void closeAdmin(HBaseAdmin admin){
method closeTable (line 103) | public static void closeTable(HTable table){
method closeResultScanner (line 112) | public static void closeResultScanner(ResultScanner resultScanner){
method convertUserStartRowkey (line 119) | public static byte[] convertUserStartRowkey(Configuration configuratio...
method convertUserEndRowkey (line 129) | public static byte[] convertUserEndRowkey(Configuration configuration) {
method convertInnerStartRowkey (line 142) | public static byte[] convertInnerStartRowkey(Configuration configurati...
method convertInnerEndRowkey (line 150) | public static byte[] convertInnerEndRowkey(Configuration configuration) {
method stringToBytes (line 159) | private static byte[] stringToBytes(String rowkey, boolean isBinaryRow...
method isRowkeyColumn (line 168) | public static boolean isRowkeyColumn(String columnName) {
method parseColumnOfNormalMode (line 176) | public static List<HbaseColumnCell> parseColumnOfNormalMode(List<Map> ...
method parseColumnOfMultiversionMode (line 215) | public static HashMap<String,HashMap<String,String>> parseColumnOfMult...
method split (line 246) | public static List<Configuration> split(Configuration configuration) {
method doSplit (line 277) | private static List<Configuration> doSplit(Configuration config, byte[...
method getEndKey (line 327) | private static String getEndKey(byte[] endRowkeyByte, byte[] regionEnd...
method getStartKey (line 350) | private static String getStartKey(byte[] startRowkeyByte, byte[] regio...
method validateParameter (line 368) | public static void validateParameter(Configuration originalConfig) {
method validateMode (line 404) | private static String validateMode(Configuration originalConfig) {
method checkMaxVersion (line 435) | private static void checkMaxVersion(Configuration configuration, Strin...
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Hbase094xReader.java
class Hbase094xReader (line 18) | public class Hbase094xReader extends Reader {
class Job (line 19) | public static class Job extends Reader.Job {
method init (line 22) | @Override
method split (line 28) | @Override
method destroy (line 34) | @Override
class Task (line 40) | public static class Task extends Reader.Task {
method init (line 44) | @Override
method prepare (line 62) | @Override
method startRead (line 71) | @Override
method post (line 94) | @Override
method destroy (line 99) | @Override
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Hbase094xReaderErrorCode.java
type Hbase094xReaderErrorCode (line 5) | public enum Hbase094xReaderErrorCode implements ErrorCode {
method Hbase094xReaderErrorCode (line 20) | private Hbase094xReaderErrorCode(String code, String description) {
method getCode (line 25) | @Override
method getDescription (line 30) | @Override
method toString (line 35) | @Override
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/HbaseAbstractTask.java
class HbaseAbstractTask (line 17) | public abstract class HbaseAbstractTask {
method HbaseAbstractTask (line 32) | public HbaseAbstractTask(com.alibaba.datax.common.util.Configuration c...
method fetchLine (line 43) | public abstract boolean fetchLine(Record record) throws Exception;
method initScan (line 46) | public abstract void initScan(Scan scan);
method prepare (line 49) | public void prepare() throws Exception {
method close (line 66) | public void close() {
method getNextHbaseRow (line 71) | protected Result getNextHbaseRow() throws IOException {
method convertBytesToAssignType (line 90) | public Column convertBytesToAssignType(ColumnType columnType, byte[] b...
method convertValueToAssignType (line 127) | public Column convertValueToAssignType(ColumnType columnType, String c...
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/HbaseColumnCell.java
class HbaseColumnCell (line 11) | public class HbaseColumnCell extends BaseObject {
method HbaseColumnCell (line 29) | private HbaseColumnCell(Builder builder) {
method getColumnType (line 63) | public ColumnType getColumnType() {
method getColumnName (line 67) | public String getColumnName() {
method getColumnFamily (line 71) | public byte[] getColumnFamily() {
method getQualifier (line 75) | public byte[] getQualifier() {
method getDateformat (line 79) | public String getDateformat() {
method getColumnValue (line 83) | public String getColumnValue() {
method isConstant (line 87) | public boolean isConstant() {
class Builder (line 92) | public static class Builder {
method Builder (line 99) | public Builder(ColumnType columnType) {
method columnName (line 103) | public Builder columnName(String columnName) {
method columnValue (line 108) | public Builder columnValue(String columnValue) {
method dateformat (line 113) | public Builder dateformat(String dateformat) {
method build (line 118) | public HbaseColumnCell build() {
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Key.java
class Key (line 3) | public final class Key {
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/ModeType.java
type ModeType (line 7) | public enum ModeType {
method ModeType (line 14) | ModeType(String mode) {
method getByTypeName (line 18) | public static ModeType getByTypeName(String modeName) {
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/MultiVersionFixedColumnTask.java
class MultiVersionFixedColumnTask (line 9) | public class MultiVersionFixedColumnTask extends MultiVersionTask {
method MultiVersionFixedColumnTask (line 11) | public MultiVersionFixedColumnTask(Configuration configuration) {
method initScan (line 15) | @Override
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/MultiVersionTask.java
class MultiVersionTask (line 19) | public abstract class MultiVersionTask extends HbaseAbstractTask {
method MultiVersionTask (line 29) | public MultiVersionTask(Configuration configuration) {
method fetchLine (line 42) | @Override
method convertCellToLine (line 71) | private void convertCellToLine(KeyValue keyValue, Record record) throw...
method setMaxVersions (line 92) | public void setMaxVersions(Scan scan) {
FILE: hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/NormalTask.java
clas
Condensed preview — 1445 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (5,592K chars).
[
{
"path": ".gitignore",
"chars": 2084,
"preview": "# Created by .ignore support plugin (hsz.mobi)\n.DS_Store\n.AppleDouble\n.LSOverride\nIcon\n._*\n.DocumentRevisions-V100\n.fsev"
},
{
"path": "NOTICE",
"chars": 2668,
"preview": "========================================================\nDataX 是阿里云 DataWorks数据集成 的开源版本，在阿里巴巴集团内被广泛使用的离线数据同步工具/平台。DataX "
},
{
"path": "README.md",
"chars": 18453,
"preview": "\n\n# DataX\n\n[![Leaderboard](https://img."
},
{
"path": "adbmysqlwriter/doc/adbmysqlwriter.md",
"chars": 9500,
"preview": "# DataX AdbMysqlWriter\n\n\n---\n\n\n## 1 快速介绍\n\nAdbMysqlWriter 插件实现了写入数据到 ADB MySQL 目的表的功能。在底层实现上， AdbMysqlWriter 通过 JDBC 连接远程"
},
{
"path": "adbmysqlwriter/pom.xml",
"chars": 2141,
"preview": "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n\txsi:schemaLoca"
},
{
"path": "adbmysqlwriter/src/main/assembly/package.xml",
"chars": 1112,
"preview": "<assembly\n\txmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n\txmlns:xsi=\"http://www.w3.org/20"
},
{
"path": "adbmysqlwriter/src/main/java/com/alibaba/datax/plugin/writer/adbmysqlwriter/AdbMysqlWriter.java",
"chars": 4749,
"preview": "package com.alibaba.datax.plugin.writer.adbmysqlwriter;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.alib"
},
{
"path": "adbmysqlwriter/src/main/resources/plugin.json",
"chars": 320,
"preview": "{\n \"name\": \"adbmysqlwriter\",\n \"class\": \"com.alibaba.datax.plugin.writer.adbmysqlwriter.AdbMysqlWriter\",\n \"descr"
},
{
"path": "adbmysqlwriter/src/main/resources/plugin_job_template.json",
"chars": 509,
"preview": "{\n \"name\": \"adbmysqlwriter\",\n \"parameter\": {\n \"username\": \"username\",\n \"password\": \"password\",\n "
},
{
"path": "adbpgwriter/pom.xml",
"chars": 3974,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\"\n xmlns:xsi=\"http://www"
},
{
"path": "adbpgwriter/src/main/assembly/package.xml",
"chars": 1100,
"preview": "<assembly\n\txmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n\txmlns:xsi=\"http://www.w3.org/20"
},
{
"path": "adbpgwriter/src/main/doc/adbpgwriter.md",
"chars": 4754,
"preview": "# DataX ADB PG Writer\n\n\n\n---\n\n\n## 1 快速介绍\nAdbpgWriter 插件实现了写入数据到 ABD PG版数据库的功能。在底层实现上，AdbpgWriter 插件会先缓存需要写入的数据，当缓存的\n数据量达"
},
{
"path": "adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/AdbpgWriter.java",
"chars": 3729,
"preview": "package com.alibaba.datax.plugin.writer.adbpgwriter;\n\nimport com.alibaba.datax.common.plugin.RecordReceiver;\nimport com."
},
{
"path": "adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/copy/Adb4pgClientProxy.java",
"chars": 7255,
"preview": "package com.alibaba.datax.plugin.writer.adbpgwriter.copy;\n\nimport com.alibaba.cloud.analyticdb.adb4pgclient.*;\nimport co"
},
{
"path": "adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/copy/AdbProxy.java",
"chars": 340,
"preview": "package com.alibaba.datax.plugin.writer.adbpgwriter.copy;\n\nimport com.alibaba.datax.common.plugin.RecordReceiver;\n\nimpor"
},
{
"path": "adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/package-info.java",
"chars": 103,
"preview": "/**\n * Greenplum Writer.\n *\n * @since 0.0.1\n */\npackage com.alibaba.datax.plugin.writer.adbpgwriter;\n\n\n"
},
{
"path": "adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Adb4pgUtil.java",
"chars": 5938,
"preview": "package com.alibaba.datax.plugin.writer.adbpgwriter.util;\n\nimport com.alibaba.cloud.analyticdb.adb4pgclient.Adb4pgClient"
},
{
"path": "adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Constant.java",
"chars": 231,
"preview": "package com.alibaba.datax.plugin.writer.adbpgwriter.util;\n/**\n * @author yuncheng\n */\npublic class Constant {\n public"
},
{
"path": "adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Key.java",
"chars": 890,
"preview": "package com.alibaba.datax.plugin.writer.adbpgwriter.util;\n/**\n * @author yuncheng\n */\npublic class Key {\n\n public fin"
},
{
"path": "adbpgwriter/src/main/resources/plugin.json",
"chars": 144,
"preview": "{\n \"name\": \"adbpgwriter\",\n \"class\": \"com.alibaba.datax.plugin.writer.adbpgwriter.AdbpgWriter\",\n \"description\": \"\",\n "
},
{
"path": "adbpgwriter/src/main/resources/plugin_job_template.json",
"chars": 196,
"preview": "{\n \"name\": \"adbpgwriter\",\n \"parameter\": {\n \"username\": \"\",\n \"password\": \"\",\n \"host\": \"\",\n \"port\": \"\",\n "
},
{
"path": "adswriter/doc/adswriter.md",
"chars": 6247,
"preview": "# DataX ADS写入\n\n\n---\n\n\n## 1 快速介绍\n\n<br />\n\n欢迎ADS加入DataX生态圈！ADSWriter插件实现了其他数据源向ADS写入功能，现有DataX所有的数据源均可以无缝接入ADS，实现数据快速导入ADS"
},
{
"path": "adswriter/pom.xml",
"chars": 4339,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\"\n xmlns:xsi=\"http://www"
},
{
"path": "adswriter/src/main/assembly/package.xml",
"chars": 1145,
"preview": "<assembly\n\txmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n\txmlns:xsi=\"http://www.w3.org/20"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsException.java",
"chars": 1323,
"preview": "package com.alibaba.datax.plugin.writer.adswriter;\n\npublic class AdsException extends Exception {\n\n private static fi"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsWriter.java",
"chars": 19006,
"preview": "package com.alibaba.datax.plugin.writer.adswriter;\n\nimport com.alibaba.datax.common.exception.DataXException;\nimport com"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/AdsWriterErrorCode.java",
"chars": 1953,
"preview": "package com.alibaba.datax.plugin.writer.adswriter;\n\nimport com.alibaba.datax.common.spi.ErrorCode;\n\npublic enum AdsWrite"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnDataType.java",
"chars": 13298,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.ads;\n\nimport java.math.BigDecimal;\nimport java.sql.Date;\nimport java.s"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java",
"chars": 1697,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.ads;\n\n/**\n * ADS column meta.<br>\n * <p>\n * select ordinal_position,co"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java",
"chars": 3628,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.ads;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * ADS t"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java",
"chars": 106,
"preview": "/**\n * ADS meta and service.\n *\n * @since 0.0.1\n */\npackage com.alibaba.datax.plugin.writer.adswriter.ads;"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsClientProxy.java",
"chars": 9394,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.insert;\n\nimport com.alibaba.cloud.analyticdb.adbclient.AdbClient;\nimpo"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertProxy.java",
"chars": 29459,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.insert;\n\nimport com.alibaba.datax.common.element.Column;\nimport com.al"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsInsertUtil.java",
"chars": 7109,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.insert;\n\nimport com.alibaba.datax.common.exception.DataXException;\nimp"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsProxy.java",
"chars": 381,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.insert;\n\nimport com.alibaba.datax.common.plugin.RecordReceiver;\n\nimpor"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/OperationType.java",
"chars": 1747,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.insert;\n\npublic enum OperationType {\n // i: insert uo:before image "
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/AdsHelper.java",
"chars": 16688,
"preview": "/**\n * \n */\npackage com.alibaba.datax.plugin.writer.adswriter.load;\n\nimport com.alibaba.datax.common.exception.DataXExce"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TableMetaHelper.java",
"chars": 3174,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.load;\n\nimport com.alibaba.datax.plugin.writer.adswriter.ads.ColumnData"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java",
"chars": 1713,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.load;\n\nimport com.alibaba.datax.common.util.Configuration;\n\n/**\n * Cre"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java",
"chars": 2072,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.odps;\n\n/**\n * ODPS 数据类型.\n * <p>\n * 当前定义了如下类型：\n * <ul>\n * <li>INTEGER\n "
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/FieldSchema.java",
"chars": 1441,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.odps;\n\n/**\n * ODPS列属性，包含列名和类型 列名和类型与SQL的DESC表或分区显示的列名和类型一致\n *\n * @sinc"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/TableMeta.java",
"chars": 3246,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.odps;\n\nimport java.util.Iterator;\nimport java.util.List;\n\n/**\n * ODPS "
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java",
"chars": 96,
"preview": "/**\n * ODPS meta.\n *\n * @since 0.0.1\n */\npackage com.alibaba.datax.plugin.writer.adswriter.odps;"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java",
"chars": 93,
"preview": "/**\n * ADS Writer.\n * \n * @since 0.0.1\n */\npackage com.alibaba.datax.plugin.writer.adswriter;"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/AdsUtil.java",
"chars": 8204,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.util;\n\nimport com.alibaba.datax.common.exception.DataXException;\nimpor"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java",
"chars": 848,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.util;\n\npublic class Constant {\n\n public static final String LOADMOD"
},
{
"path": "adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Key.java",
"chars": 1770,
"preview": "package com.alibaba.datax.plugin.writer.adswriter.util;\n\n\npublic final class Key {\n\n public final static String ADS_U"
},
{
"path": "adswriter/src/main/resources/plugin.json",
"chars": 146,
"preview": "{\n \"name\": \"adswriter\",\n \"class\": \"com.alibaba.datax.plugin.writer.adswriter.AdsWriter\",\n \"description\": \"\",\n "
},
{
"path": "adswriter/src/main/resources/plugin_job_template.json",
"chars": 236,
"preview": "{\n \"name\": \"adswriter\",\n \"parameter\": {\n \"url\": \"\",\n \"username\": \"\",\n \"password\": \"\",\n "
},
{
"path": "cassandrareader/doc/cassandrareader.md",
"chars": 3769,
"preview": "\n# CassandraReader 插件文档\n\n\n___\n\n\n\n## 1 快速介绍\n\nCassandraReader插件实现了从Cassandra读取数据。在底层实现上，CassandraReader通过datastax的java dri"
},
{
"path": "cassandrareader/pom.xml",
"chars": 3533,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "cassandrareader/src/main/assembly/package.xml",
"chars": 1338,
"preview": "<assembly\n xmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n xmlns:xsi=\"http:/"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReader.java",
"chars": 4433,
"preview": "package com.alibaba.datax.plugin.reader.cassandrareader;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.ali"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java",
"chars": 764,
"preview": "package com.alibaba.datax.plugin.reader.cassandrareader;\n\nimport com.alibaba.datax.common.spi.ErrorCode;\n\npublic enum Ca"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderHelper.java",
"chars": 18114,
"preview": "package com.alibaba.datax.plugin.reader.cassandrareader;\n\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimpo"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/Key.java",
"chars": 1400,
"preview": "package com.alibaba.datax.plugin.reader.cassandrareader;\n\n/**\n * Created by mazhenlin on 2019/8/19.\n */\npublic class Key"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings.properties",
"chars": 59,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_en_US.properties",
"chars": 0,
"preview": ""
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties",
"chars": 59,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties",
"chars": 59,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties",
"chars": 59,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF"
},
{
"path": "cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties",
"chars": 59,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF"
},
{
"path": "cassandrareader/src/main/resources/plugin.json",
"chars": 321,
"preview": "{\n \"name\": \"cassandrareader\",\n \"class\": \"com.alibaba.datax.plugin.reader.cassandrareader.CassandraReader\",\n \"de"
},
{
"path": "cassandrareader/src/main/resources/plugin_job_template.json",
"chars": 274,
"preview": "{\n \"name\": \"cassandrareader\",\n \"parameter\": {\n \"username\": \"\",\n \"password\": \"\",\n \"host\": \"\",\n"
},
{
"path": "cassandrawriter/doc/cassandrawriter.md",
"chars": 3838,
"preview": "\n# CassandraWriter 插件文档\n\n\n___\n\n\n\n## 1 快速介绍\n\nCassandraWriter插件实现了向Cassandra写入数据。在底层实现上，CassandraWriter通过datastax的java dri"
},
{
"path": "cassandrawriter/pom.xml",
"chars": 4335,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\"\n xmlns:xsi=\"http://www"
},
{
"path": "cassandrawriter/src/main/assembly/package.xml",
"chars": 1116,
"preview": "<assembly\n\txmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n\txmlns:xsi=\"http://www.w3.org/20"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriter.java",
"chars": 9280,
"preview": "package com.alibaba.datax.plugin.writer.cassandrawriter;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java",
"chars": 797,
"preview": "package com.alibaba.datax.plugin.writer.cassandrawriter;\n\nimport com.alibaba.datax.common.spi.ErrorCode;\n\n/**\n * Created"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterHelper.java",
"chars": 8993,
"preview": "package com.alibaba.datax.plugin.writer.cassandrawriter;\n\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimpo"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/Key.java",
"chars": 1490,
"preview": "package com.alibaba.datax.plugin.writer.cassandrawriter;\n\n/**\n * Created by mazhenlin on 2019/8/19.\n */\npublic class Key"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings.properties",
"chars": 136,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF.\nerrorcode.write_failed_exception=\\u5199\\u5165\\u6570\\u636E\\u"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties",
"chars": 144,
"preview": "errorcode.config_invalid_exception=Error in parameter configuration.\nerrorcode.write_failed_exception=\\u5199\\u5165\\u6570"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties",
"chars": 136,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF.\nerrorcode.write_failed_exception=\\u5199\\u5165\\u6570\\u636E\\u"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties",
"chars": 136,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF.\nerrorcode.write_failed_exception=\\u5199\\u5165\\u6570\\u636E\\u"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties",
"chars": 136,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF.\nerrorcode.write_failed_exception=\\u5199\\u5165\\u6570\\u636E\\u"
},
{
"path": "cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties",
"chars": 136,
"preview": "errorcode.config_invalid_exception=\\u914D\\u7F6E\\u9519\\u8BEF.\nerrorcode.write_failed_exception=\\u5199\\u5165\\u6570\\u636E\\u"
},
{
"path": "cassandrawriter/src/main/resources/plugin.json",
"chars": 230,
"preview": "{\n \"name\": \"cassandrawriter\",\n \"class\": \"com.alibaba.datax.plugin.writer.cassandrawriter.CassandraWriter\",\n \"de"
},
{
"path": "cassandrawriter/src/main/resources/plugin_job_template.json",
"chars": 274,
"preview": "{\n \"name\": \"cassandrawriter\",\n \"parameter\": {\n \"username\": \"\",\n \"password\": \"\",\n \"host\": \"\",\n"
},
{
"path": "clickhousereader/doc/clickhousereader.md",
"chars": 9137,
"preview": "\n# ClickhouseReader 插件文档\n\n\n___\n\n\n## 1 快速介绍\n\nClickhouseReader插件实现了从Clickhouse读取数据。在底层实现上，ClickhouseReader通过JDBC连接远程Clickh"
},
{
"path": "clickhousereader/pom.xml",
"chars": 3100,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\"\n xmlns:xsi=\"http://www"
},
{
"path": "clickhousereader/src/main/assembly/package.xml",
"chars": 1341,
"preview": "<assembly\n xmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n xmlns:xsi=\"http:/"
},
{
"path": "clickhousereader/src/main/java/com/alibaba/datax/plugin/reader/clickhousereader/ClickhouseReader.java",
"chars": 2913,
"preview": "package com.alibaba.datax.plugin.reader.clickhousereader;\n\nimport java.sql.Array;\nimport java.sql.ResultSet;\nimport java"
},
{
"path": "clickhousereader/src/main/resources/plugin.json",
"chars": 241,
"preview": "{\n \"name\": \"clickhousereader\",\n \"class\": \"com.alibaba.datax.plugin.reader.clickhousereader.ClickhouseReader\",\n \"descr"
},
{
"path": "clickhousereader/src/main/resources/plugin_job_template.json",
"chars": 334,
"preview": "{\n \"name\": \"clickhousereader\",\n \"parameter\": {\n \"username\": \"username\",\n \"password\": \"password\",\n \"column\": ["
},
{
"path": "clickhousereader/src/test/resources/basic1.json",
"chars": 1315,
"preview": "{\n \"job\": {\n \"setting\": {\n \"speed\": {\n \"channel\": 5\n }\n },\n \"content\": [\n {\n \"rea"
},
{
"path": "clickhousereader/src/test/resources/basic1.sql",
"chars": 690,
"preview": "CREATE TABLE IF NOT EXISTS default.all_type_tbl\n(\n`uint8_col` UInt8,\n`uint16_col` UInt16,\nuint32_col UInt32,\nuint64_col "
},
{
"path": "clickhousewriter/pom.xml",
"chars": 3097,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\"\n xmlns:xsi=\"http://www"
},
{
"path": "clickhousewriter/src/main/assembly/package.xml",
"chars": 1323,
"preview": "<assembly\n xmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n xmlns:xsi=\"http:/"
},
{
"path": "clickhousewriter/src/main/java/com/alibaba/datax/plugin/writer/clickhousewriter/ClickhouseWriter.java",
"chars": 10728,
"preview": "package com.alibaba.datax.plugin.writer.clickhousewriter;\n\nimport com.alibaba.datax.common.element.Column;\nimport com.al"
},
{
"path": "clickhousewriter/src/main/java/com/alibaba/datax/plugin/writer/clickhousewriter/ClickhouseWriterErrorCode.java",
"chars": 699,
"preview": "package com.alibaba.datax.plugin.writer.clickhousewriter;\n\nimport com.alibaba.datax.common.spi.ErrorCode;\n\npublic enum C"
},
{
"path": "clickhousewriter/src/main/resources/plugin.json",
"chars": 249,
"preview": "{\n \"name\": \"clickhousewriter\",\n \"class\": \"com.alibaba.datax.plugin.writer.clickhousewriter.ClickhouseWriter\",\n "
},
{
"path": "clickhousewriter/src/main/resources/plugin_job_template.json",
"chars": 517,
"preview": "{\n \"name\": \"clickhousewriter\",\n \"parameter\": {\n \"username\": \"username\",\n \"password\": \"password\",\n "
},
{
"path": "common/pom.xml",
"chars": 2661,
"preview": "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n xsi:sc"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/base/BaseObject.java",
"chars": 638,
"preview": "package com.alibaba.datax.common.base;\n\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java",
"chars": 290,
"preview": "package com.alibaba.datax.common.constant;\n\npublic final class CommonConstant {\n /**\n * 用于插件对自身 split 的每个 task 标识"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/constant/PluginType.java",
"chars": 486,
"preview": "package com.alibaba.datax.common.constant;\n\n/**\n * Created by jingxing on 14-8-31.\n */\npublic enum PluginType {\n //pl"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java",
"chars": 2332,
"preview": "package com.alibaba.datax.common.element;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java",
"chars": 2075,
"preview": "package com.alibaba.datax.common.element;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/Column.java",
"chars": 1351,
"preview": "package com.alibaba.datax.common.element;\n\nimport com.alibaba.fastjson2.JSON;\n\nimport java.math.BigDecimal;\nimport java."
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java",
"chars": 6048,
"preview": "package com.alibaba.datax.common.element;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/DateColumn.java",
"chars": 3795,
"preview": "package com.alibaba.datax.common.element;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java",
"chars": 3799,
"preview": "package com.alibaba.datax.common.element;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/LongColumn.java",
"chars": 2930,
"preview": "package com.alibaba.datax.common.element;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/OverFlowUtil.java",
"chars": 1832,
"preview": "package com.alibaba.datax.common.element;\n\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\n\nimport com.alibaba"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/Record.java",
"chars": 473,
"preview": "package com.alibaba.datax.common.element;\n\nimport java.util.Map;\n\n/**\n * Created by jingxing on 14-8-24.\n */\n\npublic int"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/element/StringColumn.java",
"chars": 3955,
"preview": "package com.alibaba.datax.common.element;\n\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.util.Da"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/exception/CommonErrorCode.java",
"chars": 1175,
"preview": "package com.alibaba.datax.common.exception;\n\nimport com.alibaba.datax.common.spi.ErrorCode;\n\n/**\n *\n */\npublic enum Comm"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/exception/DataXException.java",
"chars": 2146,
"preview": "package com.alibaba.datax.common.exception;\n\nimport com.alibaba.datax.common.spi.ErrorCode;\n\nimport java.io.PrintWriter;"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/exception/ExceptionTracker.java",
"chars": 414,
"preview": "package com.alibaba.datax.common.exception;\n\nimport java.io.PrintWriter;\nimport java.io.StringWriter;\n\npublic final clas"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/AbstractJobPlugin.java",
"chars": 558,
"preview": "package com.alibaba.datax.common.plugin;\n\n/**\n * Created by jingxing on 14-8-24.\n */\npublic abstract class AbstractJobPl"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/AbstractPlugin.java",
"chars": 2203,
"preview": "package com.alibaba.datax.common.plugin;\n\nimport com.alibaba.datax.common.base.BaseObject;\nimport com.alibaba.datax.comm"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/AbstractTaskPlugin.java",
"chars": 849,
"preview": "package com.alibaba.datax.common.plugin;\n\n/**\n * Created by jingxing on 14-8-24.\n */\npublic abstract class AbstractTaskP"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/JobPluginCollector.java",
"chars": 347,
"preview": "package com.alibaba.datax.common.plugin;\n\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * Created by jingxing on 14"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java",
"chars": 104,
"preview": "package com.alibaba.datax.common.plugin;\n\n\n/**\n * 这里只是一个标示类\n * */\npublic interface PluginCollector {\n\n}\n"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java",
"chars": 592,
"preview": "package com.alibaba.datax.common.plugin;\n\nimport com.alibaba.datax.common.util.Configuration;\n\npublic interface Pluginab"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java",
"chars": 802,
"preview": "/**\n * (C) 2010-2013 Alibaba Group Holding Limited.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java",
"chars": 892,
"preview": "/**\n * (C) 2010-2013 Alibaba Group Holding Limited.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "common/src/main/java/com/alibaba/datax/common/plugin/TaskPluginCollector.java",
"chars": 1331,
"preview": "package com.alibaba.datax.common.plugin;\n\nimport com.alibaba.datax.common.element.Record;\n\n/**\n * \n * 该接口提供给Task Plugin用"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/spi/ErrorCode.java",
"chars": 582,
"preview": "package com.alibaba.datax.common.spi;\n\n/**\n * 尤其注意：最好提供toString()实现。例如：\n * \n * <pre>\n * \n * @Override\n * public Str"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/spi/Hook.java",
"chars": 421,
"preview": "package com.alibaba.datax.common.spi;\n\nimport com.alibaba.datax.common.util.Configuration;\n\nimport java.util.Map;\n\n/**\n "
},
{
"path": "common/src/main/java/com/alibaba/datax/common/spi/Reader.java",
"chars": 1474,
"preview": "package com.alibaba.datax.common.spi;\n\nimport java.util.List;\n\nimport com.alibaba.datax.common.base.BaseObject;\nimport c"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/spi/Writer.java",
"chars": 983,
"preview": "package com.alibaba.datax.common.spi;\n\nimport com.alibaba.datax.common.base.BaseObject;\nimport com.alibaba.datax.common."
},
{
"path": "common/src/main/java/com/alibaba/datax/common/statistics/PerfRecord.java",
"chars": 6736,
"preview": "package com.alibaba.datax.common.statistics;\n\nimport com.alibaba.datax.common.util.HostUtils;\nimport org.apache.commons."
},
{
"path": "common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java",
"chars": 27757,
"preview": "package com.alibaba.datax.common.statistics;\n\nimport com.alibaba.datax.common.statistics.PerfRecord.PHASE;\nimport com.al"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/statistics/VMInfo.java",
"chars": 17067,
"preview": "package com.alibaba.datax.common.statistics;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.lang"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/Configuration.java",
"chars": 26525,
"preview": "package com.alibaba.datax.common.util;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba.da"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java",
"chars": 1237,
"preview": "package com.alibaba.datax.common.util;\n\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Set;\n\nimport or"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/DESCipher.java",
"chars": 5233,
"preview": "/**\n * (C) 2010-2022 Alibaba Group Holding Limited.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/DataXCaseEnvUtil.java",
"chars": 1526,
"preview": "package com.alibaba.datax.common.util;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n\npublic class DataXCas"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/FilterUtil.java",
"chars": 1585,
"preview": "package com.alibaba.datax.common.util;\n\nimport java.util.*;\nimport java.util.regex.Pattern;\n\n/**\n * 提供从 List<String> 中根据"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/HostUtils.java",
"chars": 1628,
"preview": "package com.alibaba.datax.common.util;\n\nimport org.apache.commons.io.IOUtils;\nimport org.slf4j.Logger;\nimport org.slf4j."
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LimitLogger.java",
"chars": 946,
"preview": "package com.alibaba.datax.common.util;\n\nimport org.apache.commons.lang3.StringUtils;\n\nimport java.util.HashMap;\nimport j"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/ListUtil.java",
"chars": 5450,
"preview": "package com.alibaba.datax.common.util;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba.da"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LocalStrings.properties",
"chars": 7883,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\n\nconfiguration.1=\\u914d\\u7f6e\\u4fe1\\u606f\\u9519\\u8bef\\uff0c\\u60a8\\u63d0\\u4f9b"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LocalStrings_en_US.properties",
"chars": 6282,
"preview": "very_like_yixiao=1{0}2{1}3\n\n\nconfiguration.1=Configuration information error. The configuration file [{0}] you provided "
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LocalStrings_ja_JP.properties",
"chars": 7864,
"preview": "very_like_yixiao=1{0}2{1}3\n\n\nconfiguration.1=\\u914d\\u7f6e\\u4fe1\\u606f\\u9519\\u8bef\\uff0c\\u60a8\\u63d0\\u4f9b\\u7684\\u914d\\u7"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LocalStrings_zh_CN.properties",
"chars": 7880,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\n\nconfiguration.1=\\u914d\\u7f6e\\u4fe1\\u606f\\u9519\\u8bef\\uff0c\\u60a8\\u63d0\\u4f9b"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LocalStrings_zh_HK.properties",
"chars": 10335,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\n\nconfiguration.1=\\u914d\\u7f6e\\u4fe1\\u606f\\u9519\\u8bef\\uff0c\\u60a8\\u63d0\\u4f9b"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LocalStrings_zh_TW.properties",
"chars": 10335,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\n\nconfiguration.1=\\u914d\\u7f6e\\u4fe1\\u606f\\u9519\\u8bef\\uff0c\\u60a8\\u63d0\\u4f9b"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/LoggerFunction.java",
"chars": 144,
"preview": "package com.alibaba.datax.common.util;\n\n/**\n * @author molin.lxd\n * @date 2021-05-09\n */\npublic interface LoggerFunction"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/MessageSource.java",
"chars": 7657,
"preview": "package com.alibaba.datax.common.util;\n\nimport java.text.MessageFormat;\nimport java.util.HashMap;\nimport java.util.Local"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/RangeSplitUtil.java",
"chars": 7932,
"preview": "package com.alibaba.datax.common.util;\n\nimport org.apache.commons.lang3.tuple.ImmutablePair;\nimport org.apache.commons.l"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/RetryUtil.java",
"chars": 7879,
"preview": "package com.alibaba.datax.common.util;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.util.List;"
},
{
"path": "common/src/main/java/com/alibaba/datax/common/util/StrUtil.java",
"chars": 3416,
"preview": "package com.alibaba.datax.common.util;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.Val"
},
{
"path": "core/pom.xml",
"chars": 5525,
"preview": "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n xsi:sc"
},
{
"path": "core/src/main/assembly/package.xml",
"chars": 3201,
"preview": "<assembly\n xmlns=\"http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0\"\n xmlns:xsi=\"http:/"
},
{
"path": "core/src/main/conf/.secret.properties",
"chars": 155,
"preview": "#ds basicAuth config\nauth.user=\nauth.pass=\ncurrent.keyVersion=\ncurrent.publicKey=\ncurrent.privateKey=\ncurrent.service.us"
},
{
"path": "core/src/main/conf/core.json",
"chars": 1652,
"preview": "\n{\n \"entry\": {\n \"jvm\": \"-Xms1G -Xmx1G\",\n \"environment\": {}\n },\n \"common\": {\n \"column\": {\n "
},
{
"path": "core/src/main/conf/logback.xml",
"chars": 6188,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<configuration>\r\n <property name=\"log.dir\" value=\"${datax.home}/log/\" />\r\n "
},
{
"path": "core/src/main/java/com/alibaba/datax/core/AbstractContainer.java",
"chars": 1006,
"preview": "package com.alibaba.datax.core;\n\nimport com.alibaba.datax.common.util.Configuration;\nimport com.alibaba.datax.core.stati"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/Engine.java",
"chars": 8328,
"preview": "package com.alibaba.datax.core;\r\n\r\nimport com.alibaba.datax.common.element.ColumnCast;\r\nimport com.alibaba.datax.common."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/LocalStrings.properties",
"chars": 282,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\nengine.1=\\u975e standalone \\u6a21\\u5f0f\\u5fc5\\u987b\\u5728 URL \\u4e2d\\u63d0\\u4"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/LocalStrings_en_US.properties",
"chars": 222,
"preview": "very_like_yixiao=1{0}2{1}3\n\nengine.1=A valid job ID must be provided in the URL for the non-standalone mode. \nengine."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/LocalStrings_ja_JP.properties",
"chars": 267,
"preview": "very_like_yixiao=1{0}2{1}3\n\nengine.1=\\u975e standalone \\u6a21\\u5f0f\\u5fc5\\u987b\\u5728 URL \\u4e2d\\u63d0\\u4f9b\\u6709\\u6548"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_CN.properties",
"chars": 282,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\nengine.1=\\u975e standalone \\u6a21\\u5f0f\\u5fc5\\u987b\\u5728 URL \\u4e2d\\u63d0\\u4"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_HK.properties",
"chars": 404,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\nengine.1=\\u975e standalone \\u6a21\\u5f0f\\u5fc5\\u987b\\u5728 URL \\u4e2d\\u63d0\\u4"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/LocalStrings_zh_TW.properties",
"chars": 404,
"preview": "very_like_yixiao=\\u4e00{0}\\u4e8c{1}\\u4e09\n\nengine.1=\\u975e standalone \\u6a21\\u5f0f\\u5fc5\\u987b\\u5728 URL \\u4e2d\\u63d0\\u4"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/container/util/HookInvoker.java",
"chars": 3160,
"preview": "package com.alibaba.datax.core.container.util;\n\n/**\n * Created by xiafei.qiuxf on 14/12/17.\n */\n\nimport com.alibaba.data"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/container/util/JobAssignUtil.java",
"chars": 7934,
"preview": "package com.alibaba.datax.core.container.util;\n\nimport com.alibaba.datax.common.constant.CommonConstant;\nimport com.alib"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/job/JobContainer.java",
"chars": 38447,
"preview": "package com.alibaba.datax.core.job;\n\nimport com.alibaba.datax.common.constant.PluginType;\nimport com.alibaba.datax.commo"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/job/meta/ExecuteMode.java",
"chars": 366,
"preview": "package com.alibaba.datax.core.job.meta;\n\n/**\n * Created by liupeng on 15/12/21.\n */\npublic enum ExecuteMode {\n STAND"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/job/meta/State.java",
"chars": 558,
"preview": "package com.alibaba.datax.core.job.meta;\n\n/**\n * Created by liupeng on 15/12/21.\n */\npublic enum State {\n SUBMITTING("
},
{
"path": "core/src/main/java/com/alibaba/datax/core/job/scheduler/AbstractScheduler.java",
"chars": 5491,
"preview": "package com.alibaba.datax.core.job.scheduler;\n\nimport com.alibaba.datax.common.exception.DataXException;\nimport com.alib"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/job/scheduler/processinner/ProcessInnerScheduler.java",
"chars": 2356,
"preview": "package com.alibaba.datax.core.job.scheduler.processinner;\n\nimport com.alibaba.datax.common.exception.DataXException;\nim"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/job/scheduler/processinner/StandAloneScheduler.java",
"chars": 495,
"preview": "package com.alibaba.datax.core.job.scheduler.processinner;\n\nimport com.alibaba.datax.core.statistics.container.communica"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/communication/Communication.java",
"chars": 7989,
"preview": "package com.alibaba.datax.core.statistics.communication;\n\nimport com.alibaba.datax.common.base.BaseObject;\nimport com.al"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/communication/CommunicationTool.java",
"chars": 12694,
"preview": "package com.alibaba.datax.core.statistics.communication;\n\nimport com.alibaba.datax.common.statistics.PerfTrace;\nimport c"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/communication/LocalTGCommunicationManager.java",
"chars": 2128,
"preview": "package com.alibaba.datax.core.statistics.communication;\n\nimport com.alibaba.datax.dataxservice.face.domain.enums.State;"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/collector/AbstractCollector.java",
"chars": 2455,
"preview": "package com.alibaba.datax.core.statistics.container.collector;\n\nimport com.alibaba.datax.common.util.Configuration;\nimpo"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java",
"chars": 507,
"preview": "package com.alibaba.datax.core.statistics.container.collector;\n\nimport com.alibaba.datax.core.statistics.communication.C"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/AbstractContainerCommunicator.java",
"chars": 2613,
"preview": "package com.alibaba.datax.core.statistics.container.communicator;\n\n\nimport com.alibaba.datax.common.statistics.VMInfo;\ni"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/job/StandAloneJobContainerCommunicator.java",
"chars": 2284,
"preview": "package com.alibaba.datax.core.statistics.container.communicator.job;\n\nimport com.alibaba.datax.common.util.Configuratio"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/AbstractTGContainerCommunicator.java",
"chars": 2518,
"preview": "package com.alibaba.datax.core.statistics.container.communicator.taskgroup;\n\nimport com.alibaba.datax.common.util.Config"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java",
"chars": 700,
"preview": "package com.alibaba.datax.core.statistics.container.communicator.taskgroup;\n\nimport com.alibaba.datax.common.util.Config"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java",
"chars": 366,
"preview": "package com.alibaba.datax.core.statistics.container.report;\n\nimport com.alibaba.datax.core.statistics.communication.Comm"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java",
"chars": 608,
"preview": "package com.alibaba.datax.core.statistics.container.report;\n\nimport com.alibaba.datax.core.statistics.communication.Comm"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/plugin/DefaultJobPluginCollector.java",
"chars": 1029,
"preview": "package com.alibaba.datax.core.statistics.plugin;\n\nimport com.alibaba.datax.common.plugin.JobPluginCollector;\nimport com"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/AbstractTaskPluginCollector.java",
"chars": 2650,
"preview": "package com.alibaba.datax.core.statistics.plugin.task;\n\nimport com.alibaba.datax.core.statistics.communication.Communica"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/HttpPluginCollector.java",
"chars": 803,
"preview": "package com.alibaba.datax.core.statistics.plugin.task;\n\nimport com.alibaba.datax.common.constant.PluginType;\nimport com."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/StdoutPluginCollector.java",
"chars": 2690,
"preview": "package com.alibaba.datax.core.statistics.plugin.task;\n\nimport com.alibaba.datax.common.constant.PluginType;\nimport com."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/util/DirtyRecord.java",
"chars": 3942,
"preview": "package com.alibaba.datax.core.statistics.plugin.task.util;\n\nimport com.alibaba.datax.common.element.Column;\nimport com."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/taskgroup/TaskGroupContainer.java",
"chars": 24060,
"preview": "package com.alibaba.datax.core.taskgroup;\n\nimport com.alibaba.datax.common.constant.PluginType;\nimport com.alibaba.datax"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/taskgroup/TaskMonitor.java",
"chars": 3826,
"preview": "package com.alibaba.datax.core.taskgroup;\n\nimport com.alibaba.datax.common.exception.CommonErrorCode;\nimport com.alibaba"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/taskgroup/runner/AbstractRunner.java",
"chars": 3082,
"preview": "package com.alibaba.datax.core.taskgroup.runner;\n\nimport com.alibaba.datax.common.plugin.AbstractTaskPlugin;\nimport com."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/taskgroup/runner/ReaderRunner.java",
"chars": 3856,
"preview": "package com.alibaba.datax.core.taskgroup.runner;\n\nimport com.alibaba.datax.common.plugin.AbstractTaskPlugin;\nimport com."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/taskgroup/runner/TaskGroupContainerRunner.java",
"chars": 1162,
"preview": "package com.alibaba.datax.core.taskgroup.runner;\n\nimport com.alibaba.datax.common.exception.DataXException;\nimport com.a"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/taskgroup/runner/WriterRunner.java",
"chars": 3505,
"preview": "package com.alibaba.datax.core.taskgroup.runner;\n\nimport com.alibaba.datax.common.plugin.AbstractTaskPlugin;\nimport com."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/channel/Channel.java",
"chars": 8803,
"preview": "package com.alibaba.datax.core.transport.channel;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.alibaba.da"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/channel/memory/MemoryChannel.java",
"chars": 3813,
"preview": "package com.alibaba.datax.core.transport.channel.memory;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.ali"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/exchanger/BufferedRecordExchanger.java",
"chars": 4368,
"preview": "package com.alibaba.datax.core.transport.exchanger;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.alibaba."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/exchanger/BufferedRecordTransformerExchanger.java",
"chars": 5601,
"preview": "package com.alibaba.datax.core.transport.exchanger;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.alibaba."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/exchanger/RecordExchanger.java",
"chars": 3622,
"preview": "/**\n * (C) 2010-2014 Alibaba Group Holding Limited.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/exchanger/TransformerExchanger.java",
"chars": 5715,
"preview": "package com.alibaba.datax.core.transport.exchanger;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.alibaba."
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/record/DefaultRecord.java",
"chars": 2679,
"preview": "package com.alibaba.datax.core.transport.record;\n\nimport com.alibaba.datax.common.element.Column;\nimport com.alibaba.dat"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java",
"chars": 914,
"preview": "package com.alibaba.datax.core.transport.record;\n\nimport com.alibaba.datax.common.element.Column;\nimport com.alibaba.dat"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/transformer/ComplexTransformerProxy.java",
"chars": 833,
"preview": "package com.alibaba.datax.core.transport.transformer;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.alibab"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/transformer/DigestTransformer.java",
"chars": 2847,
"preview": "package com.alibaba.datax.core.transport.transformer;\n\nimport com.alibaba.datax.common.element.Column;\nimport com.alibab"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java",
"chars": 9902,
"preview": "package com.alibaba.datax.core.transport.transformer;\n\nimport com.alibaba.datax.common.element.*;\nimport com.alibaba.dat"
},
{
"path": "core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformer.java",
"chars": 3495,
"preview": "package com.alibaba.datax.core.transport.transformer;\n\nimport com.alibaba.datax.common.element.Record;\nimport com.alibab"
}
]
// ... and 1245 more files (download for full content)
About this extraction
This page contains the full source code of the alibaba/DataX GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 1445 files (5.0 MB), approximately 1.4M tokens, and a symbol index with 6896 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.