lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sync): 修复字段判断空值时类型转换错误

Browse Source 
把字段值统一当作string处理，但hudi特有字段不是字符串，类型强转失败
This commit is contained in:
v-zhangjc9
2024-03-20 12:10:10 +08:00
parent 40e387239b
commit 0608fa8344
1 changed files with 78 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV2.java
View File

@@ -84,30 +84,35 @@ public class TypeConverterV2 implements TypeConverter {
} }
private Schema convertType(String table, String field, String type, Long length, Integer scala) { private Schema convertType(String table, String field, String type, Long length, Integer scala) {
type = type.trim().toLowerCase(); try {
if (BOOLEAN_REGEX.matcher(type).matches()) { type = type.trim().toLowerCase();
return NULLABLE_BOOLEAN_SCHEMA; if (BOOLEAN_REGEX.matcher(type).matches()) {
} else if (STRING_REGEX.matcher(type).matches() || DATE_REGEX.matcher(type).matches()) { return NULLABLE_BOOLEAN_SCHEMA;
return NULLABLE_STRING_SCHEMA; } else if (STRING_REGEX.matcher(type).matches() || DATE_REGEX.matcher(type).matches()) {
} else if (INT_REGEX.matcher(type).matches()) { return NULLABLE_STRING_SCHEMA;
return NULLABLE_INT_SCHEMA; } else if (INT_REGEX.matcher(type).matches()) {
} else if (LONG_REGEX.matcher(type).matches()) { return NULLABLE_INT_SCHEMA;
return NULLABLE_LONG_SCHEMA; } else if (LONG_REGEX.matcher(type).matches()) {
} else if (FLOAT_REGEX.matcher(type).matches()) { return NULLABLE_LONG_SCHEMA;
return NULLABLE_FLOAT_SCHEMA; } else if (FLOAT_REGEX.matcher(type).matches()) {
} else if (DOUBLE_REGEX.matcher(type).matches()) { return NULLABLE_FLOAT_SCHEMA;
return NULLABLE_DOUBLE_SCHEMA; } else if (DOUBLE_REGEX.matcher(type).matches()) {
} else if (FIXED_REGEX.matcher(type).matches()) { return NULLABLE_DOUBLE_SCHEMA;
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 0); } else if (FIXED_REGEX.matcher(type).matches()) {
} else if (DECIMAL_REGEX.matcher(type).matches() || NUMERIC_REGEX.matcher(type).matches()) { return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 0);
if (ObjectUtil.isNull(scala)) { } else if (DECIMAL_REGEX.matcher(type).matches() || NUMERIC_REGEX.matcher(type).matches()) {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 6); if (ObjectUtil.isNull(scala)) {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), 6);
} else {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), scala);
}
} else { } else {
return NULLABLE_DECIMAL_SCHEMA(field, length.intValue(), scala); LogHelper.warn(logger, LogHelper.LogPoint.FIELD_TYPE_NOT_FOUND, "{} Cannot find correct type for source type: {} length: {} scala: {}", table, type, length, scala);
return NULLABLE_STRING_SCHEMA;
} }
} else { } catch (Throwable throwable) {
LogHelper.warn(logger, LogHelper.LogPoint.FIELD_TYPE_NOT_FOUND, "{} Cannot find correct type for source type: {} length: {} scala: {}", table, type, length, scala); logger.error(StrUtil.format("Convert type failure {} {} {} length: {} scala: {}", table, field, type, length, scala), throwable);
return NULLABLE_STRING_SCHEMA; throw throwable;
} }
} }
@@ -118,54 +123,63 @@ public class TypeConverterV2 implements TypeConverter {
for (int index = 0; index < fields.size(); index++) { for (int index = 0; index < fields.size(); index++) {
Field field = fields.get(index); Field field = fields.get(index);
Object value = data.getOrDefault(Constants.FIELD_COVERT.apply(meta, field.name()), null); Object value = data.getOrDefault(Constants.FIELD_COVERT.apply(meta, field.name()), null);
row.setField(index, covertValue(field.schema(), value)); row.setField(index, covertValue(field.schema(), field.name(), value));
} }
return row; return row;
} }
private Object covertValue(Schema schema, Object value) { private Object covertValue(Schema schema, String name, Object value) {
if (ObjectUtil.isNull(value)) { try {
return value; if (ObjectUtil.isNull(value)) {
} else if (NULLABLE_BOOLEAN_SCHEMA.equals(schema) || BOOLEAN_SCHEMA.equals(schema)) { return value;
if (StrUtil.isBlankIfStr(value)) { } else if (NULLABLE_BOOLEAN_SCHEMA.equals(schema) || BOOLEAN_SCHEMA.equals(schema)) {
return null; if (isNullValue(value)) {
} return null;
return value instanceof String ? Boolean.valueOf((String) value) : value;
} else if (NULLABLE_INT_SCHEMA.equals(schema) || INT_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Integer.valueOf((String) value) : value;
} else if (NULLABLE_LONG_SCHEMA.equals(schema) || LONG_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Long.valueOf((String) value) : value;
} else if (NULLABLE_FLOAT_SCHEMA.equals(schema) || FLOAT_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Float.valueOf((String) value) : value;
} else if (NULLABLE_DOUBLE_SCHEMA.equals(schema) || DOUBLE_SCHEMA.equals(schema)) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
return value instanceof String ? Double.valueOf((String) value) : value;
} else if (NULLABLE_STRING_SCHEMA.equals(schema) || STRING_SCHEMA.equals(schema)) {
return StringData.fromString((String) value);
} else {
for (Schema type : schema.getTypes()) {
if (type.getLogicalType() instanceof LogicalTypes.Decimal) {
if (StrUtil.isBlankIfStr(value)) {
return null;
}
LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) type.getLogicalType();
int precision = decimalType.getPrecision();
int scala = decimalType.getScale();
return DecimalData.fromBigDecimal(new BigDecimal((String) value), precision, scala);
} }
return value instanceof String ? Boolean.valueOf((String) value) : value;
} else if (NULLABLE_INT_SCHEMA.equals(schema) || INT_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Integer.valueOf((String) value) : value;
} else if (NULLABLE_LONG_SCHEMA.equals(schema) || LONG_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Long.valueOf((String) value) : value;
} else if (NULLABLE_FLOAT_SCHEMA.equals(schema) || FLOAT_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Float.valueOf((String) value) : value;
} else if (NULLABLE_DOUBLE_SCHEMA.equals(schema) || DOUBLE_SCHEMA.equals(schema)) {
if (isNullValue(value)) {
return null;
}
return value instanceof String ? Double.valueOf((String) value) : value;
} else if (NULLABLE_STRING_SCHEMA.equals(schema) || STRING_SCHEMA.equals(schema)) {
return StringData.fromString((String) value);
} else {
for (Schema type : schema.getTypes()) {
if (type.getLogicalType() instanceof LogicalTypes.Decimal) {
if (isNullValue(value)) {
return null;
}
LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) type.getLogicalType();
int precision = decimalType.getPrecision();
int scala = decimalType.getScale();
return DecimalData.fromBigDecimal(new BigDecimal((String) value), precision, scala);
}
}
return value;
} }
return value; } catch (Throwable throwable) {
logger.error(StrUtil.format("Convert value failure {} {} {}", schema.toString(), name, value), throwable);
throw throwable;
} }
} }
private boolean isNullValue(Object value) {
return StrUtil.isBlankIfStr(value) || ObjectUtil.equals("null", value);
}
} }