wyl/DataX
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

DataX 通道能力更新（MaxCompute、Hologres、Tdengine、OSS）、安全漏洞更新、通用打包更新等

Browse Source
This commit is contained in:
jt-chen
2022-06-09 21:13:49 +08:00
parent 3225f02bf9
commit e9a579759e
261 changed files with 17980 additions and 2432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hdfswriter/pom.xml
+17 -1
View File
@@ -19,6 +19,17 @@
</properties>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.17.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.17.1</version>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-common</artifactId>
@@ -30,6 +41,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>hadoop-aliyun</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@@ -132,4 +148,4 @@
</plugin>
</plugins>
</build>
</project>
</project>
hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/HdfsHelper.java
+70
View File
@@ -6,10 +6,13 @@ import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordReceiver;
import com.alibaba.datax.common.plugin.TaskPluginCollector;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.datax.plugin.unstructuredstorage.util.ColumnTypeUtil;
import com.alibaba.datax.plugin.unstructuredstorage.util.HdfsUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
@@ -24,6 +27,10 @@ import org.apache.hadoop.mapred.*;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import parquet.schema.OriginalType;
import parquet.schema.PrimitiveType;
import parquet.schema.Types;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -556,4 +563,67 @@ public class HdfsHelper {
transportResult.setLeft(recordList);
return transportResult;
}
public static String generateParquetSchemaFromColumnAndType(List<Configuration> columns) {
Map<String, ColumnTypeUtil.DecimalInfo> decimalColInfo = new HashMap<>(16);
ColumnTypeUtil.DecimalInfo PARQUET_DEFAULT_DECIMAL_INFO = new ColumnTypeUtil.DecimalInfo(10, 2);
Types.MessageTypeBuilder typeBuilder = Types.buildMessage();
for (Configuration column : columns) {
String name = column.getString("name");
String colType = column.getString("type");
Validate.notNull(name, "column.name can't be null");
Validate.notNull(colType, "column.type can't be null");
switch (colType.toLowerCase()) {
case "tinyint":
case "smallint":
case "int":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.INT32).named(name);
break;
case "bigint":
case "long":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.INT64).named(name);
break;
case "float":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.FLOAT).named(name);
break;
case "double":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.DOUBLE).named(name);
break;
case "binary":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.BINARY).named(name);
break;
case "char":
case "varchar":
case "string":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
break;
case "boolean":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.BOOLEAN).named(name);
break;
case "timestamp":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.INT96).named(name);
break;
case "date":
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.INT32).as(OriginalType.DATE).named(name);
break;
default:
if (ColumnTypeUtil.isDecimalType(colType)) {
ColumnTypeUtil.DecimalInfo decimalInfo = ColumnTypeUtil.getDecimalInfo(colType, PARQUET_DEFAULT_DECIMAL_INFO);
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
.as(OriginalType.DECIMAL)
.precision(decimalInfo.getPrecision())
.scale(decimalInfo.getScale())
.length(HdfsUtil.computeMinBytesForPrecision(decimalInfo.getPrecision()))
.named(name);
decimalColInfo.put(name, decimalInfo);
} else {
typeBuilder.optional(PrimitiveType.PrimitiveTypeName.BINARY).named(name);
}
break;
}
}
return typeBuilder.named("m").toString();
}
}
hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/HdfsWriter.java
+49
View File
@@ -9,9 +9,11 @@ import com.google.common.collect.Sets;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import parquet.schema.MessageTypeParser;
import java.util.*;
@@ -323,8 +325,55 @@ public class HdfsWriter extends Writer {
}
return tmpFilePath;
}
public void unitizeParquetConfig(Configuration writerSliceConfig) {
String parquetSchema = writerSliceConfig.getString(Key.PARQUET_SCHEMA);
if (StringUtils.isNotBlank(parquetSchema)) {
LOG.info("parquetSchema has config. use parquetSchema:\n{}", parquetSchema);
return;
}
List<Configuration> columns = writerSliceConfig.getListConfiguration(Key.COLUMN);
if (columns == null || columns.isEmpty()) {
throw DataXException.asDataXException("parquetSchema or column can't be blank!");
}
parquetSchema = generateParquetSchemaFromColumn(columns);
// 为了兼容历史逻辑,对之前的逻辑做保留，但是如果配置的时候报错，则走新逻辑
try {
MessageTypeParser.parseMessageType(parquetSchema);
} catch (Throwable e) {
LOG.warn("The generated parquetSchema {} is illegal, try to generate parquetSchema in another way", parquetSchema);
parquetSchema = HdfsHelper.generateParquetSchemaFromColumnAndType(columns);
LOG.info("The last generated parquet schema is {}", parquetSchema);
}
writerSliceConfig.set(Key.PARQUET_SCHEMA, parquetSchema);
LOG.info("dataxParquetMode use default fields.");
writerSliceConfig.set(Key.DATAX_PARQUET_MODE, "fields");
}
private String generateParquetSchemaFromColumn(List<Configuration> columns) {
StringBuffer parquetSchemaStringBuffer = new StringBuffer();
parquetSchemaStringBuffer.append("message m {");
for (Configuration column: columns) {
String name = column.getString("name");
Validate.notNull(name, "column.name can't be null");
String type = column.getString("type");
Validate.notNull(type, "column.type can't be null");
String parquetColumn = String.format("optional %s %s;", type, name);
parquetSchemaStringBuffer.append(parquetColumn);
}
parquetSchemaStringBuffer.append("}");
String parquetSchema = parquetSchemaStringBuffer.toString();
LOG.info("generate parquetSchema:\n{}", parquetSchema);
return parquetSchema;
}
}
public static class Task extends Writer.Task {
private static final Logger LOG = LoggerFactory.getLogger(Task.class);
hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/Key.java
+13
View File
@@ -33,4 +33,17 @@ public class Key {
public static final String KERBEROS_PRINCIPAL = "kerberosPrincipal";
// hadoop config
public static final String HADOOP_CONFIG = "hadoopConfig";
// useOldRawDataTransf
public final static String PARQUET_FILE_USE_RAW_DATA_TRANSF = "useRawDataTransf";
public final static String DATAX_PARQUET_MODE = "dataxParquetMode";
// hdfs username 默认值 admin
public final static String HDFS_USERNAME = "hdfsUsername";
public static final String PROTECTION = "protection";
public static final String PARQUET_SCHEMA = "parquetSchema";
public static final String PARQUET_MERGE_RESULT = "parquetMergeResult";
}