From 4ddfc61d70e39bc09dd1aa020bde26284ed35d39 Mon Sep 17 00:00:00 2001 From: wangxianghu Date: Thu, 17 Dec 2020 18:00:23 +0800 Subject: [PATCH] [MINOR] Make QuickstartUtil generate random timestamp instead of 0 (#2340) --- .../main/java/org/apache/hudi/QuickstartUtils.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java index febdf190c..025a828dc 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java +++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java @@ -121,11 +121,22 @@ public class QuickstartUtils { */ public static OverwriteWithLatestAvroPayload generateRandomValue(HoodieKey key, String riderDriverSuffix) throws IOException { + // The timestamp generated is limited to range from 7 days before to now, to avoid generating too many + // partitionPaths when user use timestamp as partitionPath filed. GenericRecord rec = - generateGenericRecord(key.getRecordKey(), "rider-" + riderDriverSuffix, "driver-" + riderDriverSuffix, 0); + generateGenericRecord(key.getRecordKey(), "rider-" + riderDriverSuffix, "driver-" + + riderDriverSuffix, generateRangeRandomTimestamp(7)); return new OverwriteWithLatestAvroPayload(Option.of(rec)); } + /** + * Generate timestamp range from {@param daysTillNow} before to now. + */ + private static long generateRangeRandomTimestamp(int daysTillNow) { + long maxIntervalMillis = daysTillNow * 24 * 60 * 60 * 1000L; + return System.currentTimeMillis() - (long)(Math.random() * maxIntervalMillis); + } + /** * Generates new inserts, uniformly across the partition paths above. It also updates the list of existing keys. */