lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Ignore dublicate of a compaction file

Browse Source
This commit is contained in:
Alex Filipchik
2019-08-27 14:44:54 -07:00
committed by Balaji Varadarajan
parent e0ab89b3ac
commit baea4f3b82
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java
View File

@@ -150,12 +150,19 @@ public class CompactionUtils {
pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> { pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> {
return getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue()); return getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue());
}).forEach(pair -> { }).forEach(pair -> {
// Defensive check to ensure a single-fileId does not have more than one pending compaction // Defensive check to ensure a single-fileId does not have more than one pending compaction with different
// params. If we find a full dub we assume it is caused by eventual nature of the move operation on some DFSs.
if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) { if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) {
HoodieCompactionOperation operation = pair.getValue().getValue();
HoodieCompactionOperation anotherOperation =
fgIdToPendingCompactionWithInstantMap.get(pair.getKey()).getValue();
if (!operation.equals(anotherOperation)) {
String msg = "Hoodie File Id (" + pair.getKey() + ") has more thant 1 pending compactions. Instants: " String msg = "Hoodie File Id (" + pair.getKey() + ") has more thant 1 pending compactions. Instants: "
+ pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey());
throw new IllegalStateException(msg); throw new IllegalStateException(msg);
} }
}
fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue()); fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue());
}); });
return fgIdToPendingCompactionWithInstantMap; return fgIdToPendingCompactionWithInstantMap;