Incorporating code review feedback for finalizeWrite for COW #3
This commit is contained in:
@@ -569,6 +569,9 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
|
||||
.withDeletedFileResults(results).build();
|
||||
}).collect();
|
||||
|
||||
// clean temporary data files
|
||||
cleanTemporaryDataFiles(jsc);
|
||||
|
||||
// Remove the rolled back inflight commits
|
||||
commits.stream().map(s -> new HoodieInstant(true, actionType, s))
|
||||
.forEach(activeTimeline::deleteInflight);
|
||||
@@ -609,11 +612,13 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
|
||||
return new Tuple2<>(writeStat.getPath(), true);
|
||||
}).collect();
|
||||
|
||||
// clean temporary data files
|
||||
cleanTemporaryDataFiles(jsc);
|
||||
|
||||
return Optional.of(results.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cleanTemporaryDataFiles(JavaSparkContext jsc) {
|
||||
private void cleanTemporaryDataFiles(JavaSparkContext jsc) {
|
||||
if (!config.shouldUseTempFolderForCopyOnWrite()) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -281,8 +281,4 @@ public abstract class HoodieTable<T extends HoodieRecordPayload> implements Seri
|
||||
*/
|
||||
public abstract Optional<Integer> finalizeWrite(JavaSparkContext jsc, List<Tuple2<String, HoodieWriteStat>> writeStatuses);
|
||||
|
||||
/**
|
||||
* Clean temporary data files after data files are finalized or commit is rolled back.
|
||||
*/
|
||||
public abstract void cleanTemporaryDataFiles(JavaSparkContext jsc);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user