lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1551] Add support for BigDecimal and Integer when partitioning based on time. (#2851)

Browse Source 
Co-authored-by: trungchanh.le <trungchanh.le@bybit.com>
This commit is contained in:
Chanh Le
2021-04-22 21:56:20 +08:00
committed by GitHub
parent 4a3431866d
commit a1e636dc6b
2 changed files with 20 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
View File

@@ -35,6 +35,7 @@ import org.joda.time.format.DateTimeFormatter;
import java.io.IOException; import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.TimeZone; import java.util.TimeZone;
@@ -192,6 +193,10 @@ public class TimestampBasedAvroKeyGenerator extends SimpleAvroKeyGenerator {
timeMs = convertLongTimeToMillis(((Float) partitionVal).longValue()); timeMs = convertLongTimeToMillis(((Float) partitionVal).longValue());
} else if (partitionVal instanceof Long) { } else if (partitionVal instanceof Long) {
timeMs = convertLongTimeToMillis((Long) partitionVal); timeMs = convertLongTimeToMillis((Long) partitionVal);
} else if (partitionVal instanceof Integer) {
timeMs = convertLongTimeToMillis(((Integer) partitionVal).longValue());
} else if (partitionVal instanceof BigDecimal) {
timeMs = convertLongTimeToMillis(((BigDecimal) partitionVal).longValue());
} else if (partitionVal instanceof CharSequence) { } else if (partitionVal instanceof CharSequence) {
if (!inputFormatter.isPresent()) { if (!inputFormatter.isPresent()) {
throw new HoodieException("Missing inputformatter. Ensure " + Config.TIMESTAMP_INPUT_DATE_FORMAT_PROP + " config is set when timestampType is DATE_STRING or MIXED!"); throw new HoodieException("Missing inputformatter. Ensure " + Config.TIMESTAMP_INPUT_DATE_FORMAT_PROP + " config is set when timestampType is DATE_STRING or MIXED!");

15
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java
View File

@@ -36,6 +36,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.io.IOException; import java.io.IOException;
import java.math.BigDecimal;
import scala.Function1; import scala.Function1;
@@ -117,6 +118,13 @@ public class TestTimestampBasedKeyGenerator {
HoodieKey hk1 = keyGen.getKey(baseRecord); HoodieKey hk1 = keyGen.getKey(baseRecord);
assertEquals("2020-01-06 12", hk1.getPartitionPath()); assertEquals("2020-01-06 12", hk1.getPartitionPath());
// timezone is GMT+8:00, createTime is BigDecimal
baseRecord.put("createTime", new BigDecimal(1578283932000.00001));
properties = getBaseKeyConfig("EPOCHMILLISECONDS", "yyyy-MM-dd hh", "GMT+8:00", null);
keyGen = new TimestampBasedKeyGenerator(properties);
HoodieKey bigDecimalKey = keyGen.getKey(baseRecord);
assertEquals("2020-01-06 12", bigDecimalKey.getPartitionPath());
// test w/ Row // test w/ Row
baseRow = genericRecordToRow(baseRecord); baseRow = genericRecordToRow(baseRecord);
assertEquals("2020-01-06 12", keyGen.getPartitionPath(baseRow)); assertEquals("2020-01-06 12", keyGen.getPartitionPath(baseRow));
@@ -201,6 +209,13 @@ public class TestTimestampBasedKeyGenerator {
baseRow = genericRecordToRow(baseRecord); baseRow = genericRecordToRow(baseRecord);
assertEquals("1970-01-02 12", keyGen.getPartitionPath(baseRow)); assertEquals("1970-01-02 12", keyGen.getPartitionPath(baseRow));
// timezone is GMT. number of days store integer in mysql
baseRecord.put("createTime", 18736);
properties = getBaseKeyConfig("SCALAR", "yyyy-MM-dd", "GMT", "DAYS");
keyGen = new TimestampBasedKeyGenerator(properties);
HoodieKey scalarSecondsKey = keyGen.getKey(baseRecord);
assertEquals("2021-04-19", scalarSecondsKey.getPartitionPath());
} }
@Test @Test