lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Table rollback for inflight compactions MUST not delete instant files at any time to avoid race conditions

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-02-05 16:54:31 -08:00
committed by vinoth chandar
parent defcf6a0b9
commit 8adaca3454
6 changed files with 97 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
View File

@@ -347,7 +347,7 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
}
@Override
public List<HoodieRollbackStat> rollback(JavaSparkContext jsc, List<String> commits)
public List<HoodieRollbackStat> rollback(JavaSparkContext jsc, List<String> commits, boolean deleteInstants)
throws IOException {
String actionType = metaClient.getCommitActionType();
HoodieActiveTimeline activeTimeline = this.getActiveTimeline();
@@ -375,13 +375,29 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
// clean temporary data files
cleanTemporaryDataFiles(jsc);
// Remove the rolled back inflight commits
commits.stream().map(s -> new HoodieInstant(true, actionType, s))
.forEach(activeTimeline::deleteInflight);
logger.info("Deleted inflight commits " + commits);
// Delete Inflight instants if enabled
deleteInflightInstants(deleteInstants, activeTimeline,
commits.stream().map(s -> new HoodieInstant(true, actionType, s)).collect(Collectors.toList()));
return stats;
}
/**
* Delete Inflight instants if enabled
* @param deleteInstants Enable Deletion of Inflight instants
* @param activeTimeline Hoodie active timeline
* @param instantsToBeDeleted Instants to be deleted
*/
protected static void deleteInflightInstants(boolean deleteInstants, HoodieActiveTimeline activeTimeline,
List<HoodieInstant> instantsToBeDeleted) {
// Remove the rolled back inflight commits
if (deleteInstants) {
instantsToBeDeleted.forEach(activeTimeline::deleteInflight);
logger.info("Deleted inflight commits " + instantsToBeDeleted);
} else {
logger.warn("Rollback finished without deleting inflight instant files. Instants=" + instantsToBeDeleted);
}
}
/**
* Finalize the written data files
*