From a1e636dc6b4cd39606f7e47cbb1363f4fa9b79e4 Mon Sep 17 00:00:00 2001 From: Chanh Le Date: Thu, 22 Apr 2021 21:56:20 +0800 Subject: [PATCH] [HUDI-1551] Add support for BigDecimal and Integer when partitioning based on time. (#2851) Co-authored-by: trungchanh.le --- .../keygen/TimestampBasedAvroKeyGenerator.java | 5 +++++ .../keygen/TestTimestampBasedKeyGenerator.java | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java index 28048a16b..eaa10a209 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java @@ -35,6 +35,7 @@ import org.joda.time.format.DateTimeFormatter; import java.io.IOException; import java.io.Serializable; import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.TimeZone; @@ -192,6 +193,10 @@ public class TimestampBasedAvroKeyGenerator extends SimpleAvroKeyGenerator { timeMs = convertLongTimeToMillis(((Float) partitionVal).longValue()); } else if (partitionVal instanceof Long) { timeMs = convertLongTimeToMillis((Long) partitionVal); + } else if (partitionVal instanceof Integer) { + timeMs = convertLongTimeToMillis(((Integer) partitionVal).longValue()); + } else if (partitionVal instanceof BigDecimal) { + timeMs = convertLongTimeToMillis(((BigDecimal) partitionVal).longValue()); } else if (partitionVal instanceof CharSequence) { if (!inputFormatter.isPresent()) { throw new HoodieException("Missing inputformatter. Ensure " + Config.TIMESTAMP_INPUT_DATE_FORMAT_PROP + " config is set when timestampType is DATE_STRING or MIXED!"); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java index 98a8f67d6..a8390386a 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.math.BigDecimal; import scala.Function1; @@ -117,6 +118,13 @@ public class TestTimestampBasedKeyGenerator { HoodieKey hk1 = keyGen.getKey(baseRecord); assertEquals("2020-01-06 12", hk1.getPartitionPath()); + // timezone is GMT+8:00, createTime is BigDecimal + baseRecord.put("createTime", new BigDecimal(1578283932000.00001)); + properties = getBaseKeyConfig("EPOCHMILLISECONDS", "yyyy-MM-dd hh", "GMT+8:00", null); + keyGen = new TimestampBasedKeyGenerator(properties); + HoodieKey bigDecimalKey = keyGen.getKey(baseRecord); + assertEquals("2020-01-06 12", bigDecimalKey.getPartitionPath()); + // test w/ Row baseRow = genericRecordToRow(baseRecord); assertEquals("2020-01-06 12", keyGen.getPartitionPath(baseRow)); @@ -201,6 +209,13 @@ public class TestTimestampBasedKeyGenerator { baseRow = genericRecordToRow(baseRecord); assertEquals("1970-01-02 12", keyGen.getPartitionPath(baseRow)); + // timezone is GMT. number of days store integer in mysql + baseRecord.put("createTime", 18736); + properties = getBaseKeyConfig("SCALAR", "yyyy-MM-dd", "GMT", "DAYS"); + keyGen = new TimestampBasedKeyGenerator(properties); + HoodieKey scalarSecondsKey = keyGen.getKey(baseRecord); + assertEquals("2021-04-19", scalarSecondsKey.getPartitionPath()); + } @Test