lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-353] Add hive style partitioning path

Browse Source
This commit is contained in:
Wenning Ding
2019-10-30 10:41:04 -07:00
committed by Balaji Varadarajan
parent 63e330b17c
commit e555aa516d
8 changed files with 72 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java
View File

@@ -45,6 +45,8 @@ public class DayBasedCompactionStrategy extends CompactionStrategy {
// Sorts compaction in LastInFirstCompacted order
protected static Comparator<String> comparator = (String leftPartition, String rightPartition) -> {
try {
leftPartition = getPartitionPathWithoutPartitionKeys(leftPartition);
rightPartition = getPartitionPathWithoutPartitionKeys(rightPartition);
Date left = new SimpleDateFormat(datePartitionFormat, Locale.ENGLISH).parse(leftPartition);
Date right = new SimpleDateFormat(datePartitionFormat, Locale.ENGLISH).parse(rightPartition);
return left.after(right) ? -1 : right.after(left) ? 1 : 0;
@@ -77,4 +79,14 @@ public class DayBasedCompactionStrategy extends CompactionStrategy {
.collect(Collectors.toList()).subList(0, writeConfig.getTargetPartitionsPerDayBasedCompaction());
return filteredPartitionPaths;
}
/**
* If is Hive style partition path, convert it to regular partition path. e.g. year=2019/month=11/day=24 => 2019/11/24
*/
protected static String getPartitionPathWithoutPartitionKeys(String partitionPath) {
if (partitionPath.contains("=")) {
return partitionPath.replaceFirst(".*?=", "").replaceAll("/.*?=", "/");
}
return partitionPath;
}
}