From a9a97d6af47841caaa745497ec425267db0873c8 Mon Sep 17 00:00:00 2001 From: Yajun Luo <499253305@qq.com> Date: Tue, 2 Jun 2020 19:50:55 +0800 Subject: [PATCH] [HUDI-934] Add processing logic for the decimal LogicalType (#1677) --- .../hadoop/realtime/AbstractRealtimeRecordReader.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java index 3edae5cf3..5f675bbe7 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java @@ -26,6 +26,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; import org.apache.avro.generic.GenericArray; @@ -36,6 +37,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; @@ -300,6 +302,14 @@ public abstract class AbstractRealtimeRecordReader { throw new IllegalArgumentException("Only support union with null"); } case FIXED: + if (schema.getLogicalType() != null && schema.getLogicalType().getName().equals("decimal")) { + LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) LogicalTypes.fromSchema(schema); + HiveDecimalWritable writable = new HiveDecimalWritable(((GenericFixed) value).bytes(), + decimal.getScale()); + return HiveDecimalWritable.enforcePrecisionScale(writable, + decimal.getPrecision(), + decimal.getScale()); + } return new BytesWritable(((GenericFixed) value).bytes()); default: return null;