diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java index dfecf28eb..0588606fe 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java @@ -150,11 +150,18 @@ public class CompactionUtils { pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> { return getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue()); }).forEach(pair -> { - // Defensive check to ensure a single-fileId does not have more than one pending compaction + // Defensive check to ensure a single-fileId does not have more than one pending compaction with different + // params. If we find a full dub we assume it is caused by eventual nature of the move operation on some DFSs. if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) { - String msg = "Hoodie File Id (" + pair.getKey() + ") has more thant 1 pending compactions. Instants: " - + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); - throw new IllegalStateException(msg); + HoodieCompactionOperation operation = pair.getValue().getValue(); + HoodieCompactionOperation anotherOperation = + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()).getValue(); + + if (!operation.equals(anotherOperation)) { + String msg = "Hoodie File Id (" + pair.getKey() + ") has more thant 1 pending compactions. Instants: " + + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); + throw new IllegalStateException(msg); + } } fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue()); });