lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sync): 修复decimal类型名称重复导致schema解析报错

Browse Source 
Schema中的类型名称不能重复，比如有一个decimal_15如果后面的字段再出现同样的类型名称，就会报错
This commit is contained in:
兰缘小妖
2024-03-04 09:08:46 +08:00
parent c1d423bc32
commit 299d8d5e23
3 changed files with 42 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV2.java
View File

@@ -1,6 +1,7 @@
package com.lanyuanxiaoyao.service.sync.functions.type;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.common.Constants;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import java.math.BigDecimal;
@@ -34,8 +35,8 @@ public class TypeConverterV2 implements TypeConverter {
public static final Schema FLOAT_SCHEMA = create(Type.FLOAT);
public static final Schema DOUBLE_SCHEMA = create(Type.DOUBLE);
public static final Schema STRING_SCHEMA = create(Type.STRING);
public static final Function<Integer, Schema> FIXED_SCHEMA = length -> createFixed("decimal_" + length, null, null, length);
public static final BiFunction<Integer, Integer, Schema> DECIMAL_SCHEMA = (length, scala) -> LogicalTypes.decimal(length, scala).addToSchema(FIXED_SCHEMA.apply(length));
public static final BiFunction<Integer, Integer, Schema> FIXED_SCHEMA = (length, scala) -> createFixed(StrUtil.format("decimal_{}_{}", length, scala), null, null, length);
public static final BiFunction<Integer, Integer, Schema> DECIMAL_SCHEMA = (length, scala) -> LogicalTypes.decimal(length, scala).addToSchema(FIXED_SCHEMA.apply(length, scala));
public static final BiFunction<Integer, Integer, Schema> NULLABLE_DECIMAL_SCHEMA = (length, scala) -> createUnion(NULL_SCHEMA, DECIMAL_SCHEMA.apply(length, scala));
public static final Schema NULLABLE_BOOLEAN_SCHEMA = createUnion(NULL_SCHEMA, BOOLEAN_SCHEMA);
public static final Schema NULLABLE_INT_SCHEMA = createUnion(NULL_SCHEMA, INT_SCHEMA);
@@ -43,7 +44,6 @@ public class TypeConverterV2 implements TypeConverter {
public static final Schema NULLABLE_FLOAT_SCHEMA = createUnion(NULL_SCHEMA, FLOAT_SCHEMA);
public static final Schema NULLABLE_DOUBLE_SCHEMA = createUnion(NULL_SCHEMA, DOUBLE_SCHEMA);
public static final Schema NULLABLE_STRING_SCHEMA = createUnion(NULL_SCHEMA, STRING_SCHEMA);
public static final Function<Integer, Schema> NULLABLE_FIXED_SCHEMA = length -> createUnion(NULL_SCHEMA, FIXED_SCHEMA.apply(length));
private static final Pattern BOOLEAN_REGEX = Pattern.compile("^boolean|bool$");
private static final Pattern INT_REGEX = Pattern.compile("^(tinyint|smallint|int|smallserial|integer)(\\(\\d+\\))?$");
private static final Pattern LONG_REGEX = Pattern.compile("^(bigint unsigned)|((bigint|serial|long)(\\(\\d+\\))?)$");

28
utils/sync/src/test/java/com/lanyuanxiaoyao/service/sync/TestType.java Normal file
View File

@@ -0,0 +1,28 @@
package com.lanyuanxiaoyao.service.sync;
import cn.hutool.http.HttpUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import com.lanyuanxiaoyao.service.sync.functions.type.TypeConverterV2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author lanyuanxiaoyao
* @date 2024-03-01
*/
public class TestType {
private static final Logger logger = LoggerFactory.getLogger(TestType.class);
public static void main(String[] args) throws JsonProcessingException {
ObjectMapper mapper = new ObjectMapper();
String content = HttpUtil
.createGet("http://b12s7.hdp.dc:15123/info/table_meta/detail?flink_job_id=1749270121369489408&alias=irms_main_pr_type_type")
.basicAuth("AxhEbscwsJDbYMH2", "cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4")
.execute()
.body();
TableMeta meta = mapper.readValue(content, TableMeta.class);
logger.info("{}", new TypeConverterV2().convertToSchema(meta).toString());
}
}

11
utils/sync/src/test/resources/logback.xml Normal file
View File

@@ -0,0 +1,11 @@
<configuration>
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %p [%t] %logger #@# %m%n%ex{full}</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="Console"/>
</root>
</configuration>