getTypes() {
return TYPES;
}
/**
* Get the name of the Java class for the given value type.
*
* @param type the value type
* @return the class name
*/
public static String getTypeClassName(int type) {
switch (type) {
case BOOLEAN:
// "java.lang.Boolean";
return Boolean.class.getName();
case BYTE:
// "java.lang.Byte";
return Byte.class.getName();
case SHORT:
// "java.lang.Short";
return Short.class.getName();
case INT:
// "java.lang.Integer";
return Integer.class.getName();
case LONG:
// "java.lang.Long";
return Long.class.getName();
case DECIMAL:
// "java.math.BigDecimal";
return BigDecimal.class.getName();
case TIME:
// "java.sql.Time";
return Time.class.getName();
case DATE:
// "java.sql.Date";
return Date.class.getName();
case TIMESTAMP:
// "java.sql.Timestamp";
return Timestamp.class.getName();
case STRING:
// case STRING_IGNORECASE:
// case STRING_FIXED:
case MULTI_VALUE:
// "java.lang.String";
return String.class.getName();
case DOUBLE:
// "java.lang.Double";
return Double.class.getName();
case FLOAT:
// "java.lang.Float";
return Float.class.getName();
// case NULL:
// return null;
default:
throw new IllegalArgumentException("type=" + type);
}
}
/**
* Get the data type object for the given value type.
*
* @param type the value type
* @return the data type object
*/
public static ColumnDataType getDataType(int type) {
if (type < 0 || type >= TYPE_COUNT) {
throw new IllegalArgumentException("type=" + type);
}
ColumnDataType dt = TYPES_BY_VALUE_TYPE.get(type);
// if (dt == null) {
// dt = TYPES_BY_VALUE_TYPE.get(NULL);
// }
return dt;
}
/**
* Convert a value type to a SQL type.
*
* @param type the value type
* @return the SQL type
*/
public static int convertTypeToSQLType(int type) {
return getDataType(type).sqlType;
}
/**
* Convert a SQL type to a value type.
*
* @param sqlType the SQL type
* @return the value type
*/
public static int convertSQLTypeToValueType(int sqlType) {
switch (sqlType) {
// case Types.CHAR:
// case Types.NCHAR:
// return STRING_FIXED;
case Types.VARCHAR:
case Types.LONGVARCHAR:
case Types.NVARCHAR:
case Types.LONGNVARCHAR:
return STRING;
case Types.NUMERIC:
case Types.DECIMAL:
return DECIMAL;
case Types.BIT:
case Types.BOOLEAN:
return BOOLEAN;
case Types.INTEGER:
return INT;
case Types.SMALLINT:
return SHORT;
case Types.TINYINT:
return BYTE;
case Types.BIGINT:
return LONG;
case Types.REAL:
return FLOAT;
case Types.DOUBLE:
case Types.FLOAT:
return DOUBLE;
case Types.DATE:
return DATE;
case Types.TIME:
return TIME;
case Types.TIMESTAMP:
return TIMESTAMP;
// case Types.NULL:
// return NULL;
default:
throw new IllegalArgumentException("JDBC Type: " + sqlType);
}
}
/**
* Get the value type for the given Java class.
*
* @param x the Java class
* @return the value type
*/
public static int getTypeFromClass(Class x) {
// if (x == null || Void.TYPE == x) {
// return NULL;
// }
if (x.isPrimitive()) {
x = getNonPrimitiveClass(x);
}
if (String.class == x) {
return STRING;
} else if (Integer.class == x) {
return INT;
} else if (Long.class == x) {
return LONG;
} else if (Boolean.class == x) {
return BOOLEAN;
} else if (Double.class == x) {
return DOUBLE;
} else if (Byte.class == x) {
return BYTE;
} else if (Short.class == x) {
return SHORT;
} else if (Float.class == x) {
return FLOAT;
// } else if (Void.class == x) {
// return NULL;
} else if (BigDecimal.class.isAssignableFrom(x)) {
return DECIMAL;
} else if (Date.class.isAssignableFrom(x)) {
return DATE;
} else if (Time.class.isAssignableFrom(x)) {
return TIME;
} else if (Timestamp.class.isAssignableFrom(x)) {
return TIMESTAMP;
} else if (java.util.Date.class.isAssignableFrom(x)) {
return TIMESTAMP;
} else {
throw new IllegalArgumentException("class=" + x);
}
}
/**
* Convert primitive class names to java.lang.* class names.
*
* @param clazz the class (for example: int)
* @return the non-primitive class (for example: java.lang.Integer)
*/
public static Class getNonPrimitiveClass(Class clazz) {
if (!clazz.isPrimitive()) {
return clazz;
} else if (clazz == boolean.class) {
return Boolean.class;
} else if (clazz == byte.class) {
return Byte.class;
} else if (clazz == char.class) {
return Character.class;
} else if (clazz == double.class) {
return Double.class;
} else if (clazz == float.class) {
return Float.class;
} else if (clazz == int.class) {
return Integer.class;
} else if (clazz == long.class) {
return Long.class;
} else if (clazz == short.class) {
return Short.class;
} else if (clazz == void.class) {
return Void.class;
}
return clazz;
}
/**
* Get a data type object from a type name.
*
* @param s the type name
* @return the data type object
*/
public static ColumnDataType getTypeByName(String s) {
return TYPES_BY_NAME.get(s);
}
/**
* Check if the given value type is a String (VARCHAR,...).
*
* @param type the value type
* @return true if the value type is a String type
*/
public static boolean isStringType(int type) {
if (type == STRING /* || type == STRING_FIXED || type == STRING_IGNORECASE */
|| type == MULTI_VALUE) {
return true;
}
return false;
}
/**
* @return
*/
public boolean supportsAdd() {
return supportsAdd(type);
}
/**
* Check if the given value type supports the add operation.
*
* @param type the value type
* @return true if add is supported
*/
public static boolean supportsAdd(int type) {
switch (type) {
case BYTE:
case DECIMAL:
case DOUBLE:
case FLOAT:
case INT:
case LONG:
case SHORT:
return true;
default:
return false;
}
}
/**
* Get the data type that will not overflow when calling 'add' 2 billion times.
*
* @param type the value type
* @return the data type that supports adding
*/
public static int getAddProofType(int type) {
switch (type) {
case BYTE:
return LONG;
case FLOAT:
return DOUBLE;
case INT:
return LONG;
case LONG:
return DECIMAL;
case SHORT:
return LONG;
default:
return type;
}
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/ColumnInfo.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.ads;
/**
* ADS column meta.
*
* select ordinal_position,column_name,data_type,type_name,column_comment
* from information_schema.columns
* where table_schema='db_name' and table_name='table_name'
* and is_deleted=0
* order by ordinal_position limit 1000
*
*
* @since 0.0.1
*/
public class ColumnInfo {
private int ordinal;
private String name;
private ColumnDataType dataType;
private boolean isDeleted;
private String comment;
public int getOrdinal() {
return ordinal;
}
public void setOrdinal(int ordinal) {
this.ordinal = ordinal;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public ColumnDataType getDataType() {
return dataType;
}
public void setDataType(ColumnDataType dataType) {
this.dataType = dataType;
}
public boolean isDeleted() {
return isDeleted;
}
public void setDeleted(boolean isDeleted) {
this.isDeleted = isDeleted;
}
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("ColumnInfo [ordinal=").append(ordinal).append(", name=").append(name).append(", dataType=")
.append(dataType).append(", isDeleted=").append(isDeleted).append(", comment=").append(comment)
.append("]");
return builder.toString();
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/TableInfo.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.ads;
import java.util.ArrayList;
import java.util.List;
/**
* ADS table meta.
*
* select table_schema, table_name,comments
* from information_schema.tables
* where table_schema='alimama' and table_name='click_af' limit 1
*
*
* select ordinal_position,column_name,data_type,type_name,column_comment
* from information_schema.columns
* where table_schema='db_name' and table_name='table_name'
* and is_deleted=0
* order by ordinal_position limit 1000
*
*
* @since 0.0.1
*/
public class TableInfo {
private String tableSchema;
private String tableName;
private List columns;
private String comments;
private String tableType;
private String updateType;
private String partitionType;
private String partitionColumn;
private int partitionCount;
private List primaryKeyColumns;
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("TableInfo [tableSchema=").append(tableSchema).append(", tableName=").append(tableName)
.append(", columns=").append(columns).append(", comments=").append(comments).append(",updateType=").append(updateType)
.append(",partitionType=").append(partitionType).append(",partitionColumn=").append(partitionColumn).append(",partitionCount=").append(partitionCount)
.append(",primaryKeyColumns=").append(primaryKeyColumns).append("]");
return builder.toString();
}
public String getTableSchema() {
return tableSchema;
}
public void setTableSchema(String tableSchema) {
this.tableSchema = tableSchema;
}
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public List getColumns() {
return columns;
}
public List getColumnsNames() {
List columnNames = new ArrayList();
for (ColumnInfo column : this.getColumns()) {
columnNames.add(column.getName());
}
return columnNames;
}
public void setColumns(List columns) {
this.columns = columns;
}
public String getComments() {
return comments;
}
public void setComments(String comments) {
this.comments = comments;
}
public String getTableType() {
return tableType;
}
public void setTableType(String tableType) {
this.tableType = tableType;
}
public String getUpdateType() {
return updateType;
}
public void setUpdateType(String updateType) {
this.updateType = updateType;
}
public String getPartitionType() {
return partitionType;
}
public void setPartitionType(String partitionType) {
this.partitionType = partitionType;
}
public String getPartitionColumn() {
return partitionColumn;
}
public void setPartitionColumn(String partitionColumn) {
this.partitionColumn = partitionColumn;
}
public int getPartitionCount() {
return partitionCount;
}
public void setPartitionCount(int partitionCount) {
this.partitionCount = partitionCount;
}
public List getPrimaryKeyColumns() {
return primaryKeyColumns;
}
public void setPrimaryKeyColumns(List primaryKeyColumns) {
this.primaryKeyColumns = primaryKeyColumns;
}
}
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java
================================================
/**
* ADS meta and service.
*
* @since 0.0.1
*/
package com.alibaba.datax.plugin.writer.adswriter.ads;
================================================
FILE: adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsClientProxy.java
================================================
package com.alibaba.datax.plugin.writer.adswriter.insert;
import com.alibaba.cloud.analyticdb.adbclient.AdbClient;
import com.alibaba.cloud.analyticdb.adbclient.AdbClientException;
import com.alibaba.cloud.analyticdb.adbclient.DatabaseConfig;
import com.alibaba.cloud.analyticdb.adbclient.Row;
import com.alibaba.datax.common.element.Column;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.element.StringColumn;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.TaskPluginCollector;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.core.transport.record.DefaultRecord;
import com.alibaba.datax.plugin.rdbms.util.DBUtil;
import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.AdsWriterErrorCode;
import com.alibaba.datax.plugin.writer.adswriter.ads.TableInfo;
import com.alibaba.datax.plugin.writer.adswriter.util.Constant;
import com.alibaba.datax.plugin.writer.adswriter.util.Key;
import com.alibaba.fastjson2.JSON;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.Types;
import java.util.*;
public class AdsClientProxy implements AdsProxy {
private static final Logger LOG = LoggerFactory.getLogger(AdsClientProxy.class);
private String table;
private TaskPluginCollector taskPluginCollector;
public Configuration configuration;
// columnName:
private Map> adsTableColumnsMetaData;
private Map> userConfigColumnsMetaData;
private boolean useRawData[];
private AdbClient adbClient;
/**
* warn: not support columns as *
*/
public AdsClientProxy(String table, List columns, Configuration configuration,
TaskPluginCollector taskPluginCollector, TableInfo tableInfo) {
this.configuration = configuration;
this.taskPluginCollector = taskPluginCollector;
this.adsTableColumnsMetaData = AdsInsertUtil.getColumnMetaData(tableInfo, columns);
this.userConfigColumnsMetaData = new HashMap>();
List adsColumnsNames = tableInfo.getColumnsNames();
// 要使用用户配置的column顺序
this.useRawData = new boolean[columns.size()];
for (int i = 0; i < columns.size(); i++) {
String oriEachColumn = columns.get(i);
String eachColumn = oriEachColumn;
// 防御性保留字
if (eachColumn.startsWith(Constant.ADS_QUOTE_CHARACTER)
&& eachColumn.endsWith(Constant.ADS_QUOTE_CHARACTER)) {
eachColumn = eachColumn.substring(1, eachColumn.length() - 1);
}
for (String eachAdsColumn : adsColumnsNames) {
if (eachColumn.equalsIgnoreCase(eachAdsColumn)) {
Pair eachMeta = this.adsTableColumnsMetaData.get(eachAdsColumn);
this.userConfigColumnsMetaData.put(oriEachColumn, eachMeta);
int columnSqltype = eachMeta.getLeft();
switch (columnSqltype) {
case Types.DATE:
case Types.TIME:
case Types.TIMESTAMP:
this.useRawData[i] = false;
break;
default:
this.useRawData[i] = true;
break;
}
}
}
}
DatabaseConfig databaseConfig = new DatabaseConfig();
String url = configuration.getString(Key.ADS_URL);
String[] hostAndPort = StringUtils.split(url, ":");
if (hostAndPort.length != 2) {
throw DataXException.asDataXException(AdsWriterErrorCode.INVALID_CONFIG_VALUE,
"url should be in host:port format!");
}
this.table = table.toLowerCase();
databaseConfig.setHost(hostAndPort[0]);
databaseConfig.setPort(Integer.parseInt(hostAndPort[1]));
databaseConfig.setUser(configuration.getString(Key.USERNAME));
databaseConfig.setPassword(configuration.getString(Key.PASSWORD));
databaseConfig.setDatabase(configuration.getString(Key.SCHEMA));
databaseConfig.setTable(Collections.singletonList(this.table));
databaseConfig.setColumns(this.table, columns);
// 如果出现insert失败，是否跳过
boolean ignoreInsertError = configuration.getBool("ignoreInsertError", false);
databaseConfig.setIgnoreInsertError(ignoreInsertError);
// If the value of column is empty, set null
boolean emptyAsNull = configuration.getBool(Key.EMPTY_AS_NULL, false);
databaseConfig.setEmptyAsNull(emptyAsNull);
// 使用insert ignore into方式进行插入
boolean ignoreInsert = configuration.getBool(Key.IGNORE_INSERT, false);
databaseConfig.setInsertIgnore(ignoreInsert);
// commit时，写入ADB出现异常时重试的3次
int retryTimes = configuration.getInt(Key.RETRY_CONNECTION_TIME, Constant.DEFAULT_RETRY_TIMES);
databaseConfig.setRetryTimes(retryTimes);
// 重试间隔的时间为1s，单位是ms
int retryIntervalTime = configuration.getInt(Key.RETRY_INTERVAL_TIME, 1000);
databaseConfig.setRetryIntervalTime(retryIntervalTime);
// 设置自动提交的SQL长度（单位Byte），默认为32KB，一般不建议设置
int commitSize = configuration.getInt("commitSize", 32768);
databaseConfig.setCommitSize(commitSize);
// sdk默认为true
boolean partitionBatch = configuration.getBool("partitionBatch", true);
databaseConfig.setPartitionBatch(partitionBatch);
// 设置写入adb时的并发线程数，默认4，针对配置的所有表
int parallelNumber = configuration.getInt("parallelNumber", 4);
databaseConfig.setParallelNumber(parallelNumber);
// 设置client中使用的logger对象，此处使用slf4j.Logger
databaseConfig.setLogger(AdsClientProxy.LOG);
// 设置在拼接insert sql时是否需要带上字段名，默认为true
boolean insertWithColumnName = configuration.getBool("insertWithColumnName", true);
databaseConfig.setInsertWithColumnName(insertWithColumnName);
// sdk 默认值为true
boolean shareDataSource = configuration.getBool("shareDataSource", true);
databaseConfig.setShareDataSource(shareDataSource);
String password = databaseConfig.getPassword();
databaseConfig.setPassword(password.replaceAll(".", "*"));
// 避免敏感信息直接打印
LOG.info("Adb database config is : {}", JSON.toJSONString(databaseConfig));
databaseConfig.setPassword(password);
// Initialize AdbClient，初始化实例之后，databaseConfig的配置信息不能再修改
this.adbClient = new AdbClient(databaseConfig);
}
@Override
public void startWriteWithConnection(RecordReceiver recordReceiver, Connection connection, int columnNumber) {
try {
Record record;
while ((record = recordReceiver.getFromReader()) != null) {
Row row = new Row();
List