lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1435] Fix bug in Marker File Reconciliation for Non-Partitioned datasets (#2301)

Browse Source
This commit is contained in:
Balaji Varadarajan
2020-12-14 22:24:12 -08:00
committed by GitHub
parent facde4c16f
commit 069a1dcf24
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
View File

@@ -460,7 +460,7 @@ public abstract class HoodieTable<T extends HoodieRecordPayload, I, K, O> implem
if (!invalidDataPaths.isEmpty()) { if (!invalidDataPaths.isEmpty()) {
LOG.info("Removing duplicate data files created due to spark retries before committing. Paths=" + invalidDataPaths); LOG.info("Removing duplicate data files created due to spark retries before committing. Paths=" + invalidDataPaths);
Map<String, List<Pair<String, String>>> invalidPathsByPartition = invalidDataPaths.stream() Map<String, List<Pair<String, String>>> invalidPathsByPartition = invalidDataPaths.stream()
.map(dp -> Pair.of(new Path(dp).getParent().toString(), new Path(basePath, dp).toString())) .map(dp -> Pair.of(new Path(basePath, dp).getParent().toString(), new Path(basePath, dp).toString()))
.collect(Collectors.groupingBy(Pair::getKey)); .collect(Collectors.groupingBy(Pair::getKey));
// Ensure all files in delete list is actually present. This is mandatory for an eventually consistent FS. // Ensure all files in delete list is actually present. This is mandatory for an eventually consistent FS.