lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sync): 修复字段判断空值时类型转换错误

Browse Source 
把字段值统一当作string处理，但hudi特有字段不是字符串，类型强转失败
This commit is contained in:
v-zhangjc9
2024-03-20 12:10:10 +08:00
parent 40e387239b
commit 0608fa8344
1 changed files with 78 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV2.java
View File

@@ -84,30 +84,35 @@ public class TypeConverterV2 implements TypeConverter {
}
private Schema convertType(String table, String field, String type, Long length, Integer scala) {
type = type.trim().toLowerCase();
if (BOOLEAN_REGEX.matcher(type).matches()) {
return NULLABLE_BOOLEAN_SCHEMA;
} else if (STRING_REGEX.matcher(type).matches() || DATE_REGEX.matcher(type).matches()) {
return NULLABLE_STRING_SCHEMA;
} else if (INT_REGEX.matcher(type).matches()) {
return NULLABLE_INT_SCHEMA;
} else if (LONG_REGEX.matcher(type).matches()) {
return NULLABLE_LONG_SCHEMA;
} else if (FLOAT_REGEX.matcher(type).matches()) {
return NULLABLE_FLOAT_SCHEMA;
} else if (DOUBLE_REGEX.matcher(type).matches()) {
return NULLABLE_DOUBLE_SCHEMA;
} else if (FIXED_REGEX.matcher(type).matches()) {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 0);
} else if (DECIMAL_REGEX.matcher(type).matches() || NUMERIC_REGEX.matcher(type).matches()) {
if (ObjectUtil.isNull(scala)) {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 6);
try {
type = type.trim().toLowerCase();
if (BOOLEAN_REGEX.matcher(type).matches()) {
return NULLABLE_BOOLEAN_SCHEMA;
} else if (STRING_REGEX.matcher(type).matches() || DATE_REGEX.matcher(type).matches()) {
return NULLABLE_STRING_SCHEMA;
} else if (INT_REGEX.matcher(type).matches()) {
return NULLABLE_INT_SCHEMA;
} else if (LONG_REGEX.matcher(type).matches()) {
return NULLABLE_LONG_SCHEMA;
} else if (FLOAT_REGEX.matcher(type).matches()) {
return NULLABLE_FLOAT_SCHEMA;
} else if (DOUBLE_REGEX.matcher(type).matches()) {
return NULLABLE_DOUBLE_SCHEMA;
} else if (FIXED_REGEX.matcher(type).matches()) {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 0);
} else if (DECIMAL_REGEX.matcher(type).matches() || NUMERIC_REGEX.matcher(type).matches()) {
if (ObjectUtil.isNull(scala)) {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 6);
} else {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), scala);
}
} else {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), scala);
LogHelper.warn(logger, LogHelper.LogPoint.FIELD_TYPE_NOT_FOUND, "{} Cannot find correct type for source type: {} length: {} scala: {}", table, type, length, scala);
return NULLABLE_STRING_SCHEMA;
}
} else {
LogHelper.warn(logger, LogHelper.LogPoint.FIELD_TYPE_NOT_FOUND, "{} Cannot find correct type for source type: {} length: {} scala: {}", table, type, length, scala);
return NULLABLE_STRING_SCHEMA;
} catch (Throwable throwable) {
logger.error(StrUtil.format("Convert type failure {} {} {} length: {} scala: {}", table, field, type, length, scala), throwable);
throw throwable;
}
}
@@ -118,54 +123,63 @@ public class TypeConverterV2 implements TypeConverter {
for (int index = 0; index < fields.size(); index++) {
Field field = fields.get(index);
Object value = data.getOrDefault(Constants.FIELD_COVERT.apply(meta, field.name()), null);
row.setField(index, covertValue(field.schema(), value));
row.setField(index, covertValue(field.schema(), field.name(), value));
}
return row;
}
private Object covertValue(Schema schema, Object value) {
if (ObjectUtil.isNull(value)) {
return value;
} else if (NULLABLE_BOOLEAN_SCHEMA.equals(schema) || BOOLEAN_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Boolean.valueOf((String) value) : value;
} else if (NULLABLE_INT_SCHEMA.equals(schema) || INT_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Integer.valueOf((String) value) : value;
} else if (NULLABLE_LONG_SCHEMA.equals(schema) || LONG_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Long.valueOf((String) value) : value;
} else if (NULLABLE_FLOAT_SCHEMA.equals(schema) || FLOAT_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Float.valueOf((String) value) : value;
} else if (NULLABLE_DOUBLE_SCHEMA.equals(schema) || DOUBLE_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Double.valueOf((String) value) : value;
} else if (NULLABLE_STRING_SCHEMA.equals(schema) || STRING_SCHEMA.equals(schema)) {
return StringData.fromString((String) value);
} else {
for (Schema type : schema.getTypes()) {
if (type.getLogicalType() instanceof LogicalTypes.Decimal) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) type.getLogicalType();
int precision = decimalType.getPrecision();
int scala = decimalType.getScale();
return DecimalData.fromBigDecimal(new BigDecimal((String) value), precision, scala);
private Object covertValue(Schema schema, String name, Object value) {
try {
if (ObjectUtil.isNull(value)) {
return value;
} else if (NULLABLE_BOOLEAN_SCHEMA.equals(schema) || BOOLEAN_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Boolean.valueOf((String) value) : value;
} else if (NULLABLE_INT_SCHEMA.equals(schema) || INT_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Integer.valueOf((String) value) : value;
} else if (NULLABLE_LONG_SCHEMA.equals(schema) || LONG_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Long.valueOf((String) value) : value;
} else if (NULLABLE_FLOAT_SCHEMA.equals(schema) || FLOAT_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Float.valueOf((String) value) : value;
} else if (NULLABLE_DOUBLE_SCHEMA.equals(schema) || DOUBLE_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Double.valueOf((String) value) : value;
} else if (NULLABLE_STRING_SCHEMA.equals(schema) || STRING_SCHEMA.equals(schema)) {
return StringData.fromString((String) value);
} else {
for (Schema type : schema.getTypes()) {
if (type.getLogicalType() instanceof LogicalTypes.Decimal) {
if (isNullValue(value)) {
return null;
}
LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) type.getLogicalType();
int precision = decimalType.getPrecision();
int scala = decimalType.getScale();
return DecimalData.fromBigDecimal(new BigDecimal((String) value), precision, scala);
}
}
return value;
}
return value;
} catch (Throwable throwable) {
logger.error(StrUtil.format("Convert value failure {} {} {}", schema.toString(), name, value), throwable);
throw throwable;
}
}
private boolean isNullValue(Object value) {
return StrUtil.isBlankIfStr(value) || ObjectUtil.equals("null", value);
}
}