lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2715] The BitCaskDiskMap iterator may cause memory leak (#3951)

Browse Source
This commit is contained in:
Danny Chan
2021-11-09 15:40:00 +08:00
committed by GitHub
parent 6d109c6de5
commit e057a10499
5 changed files with 47 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
View File

@@ -181,6 +181,7 @@ public abstract class HoodieCompactor<T extends HoodieRecordPayload, I, K, O> im
.withBitCaskDiskMapCompressionEnabled(config.getCommonConfig().isBitCaskDiskMapCompressionEnabled())
.build();
if (!scanner.iterator().hasNext()) {
scanner.close();
return new ArrayList<>();
}
@@ -198,6 +199,7 @@ public abstract class HoodieCompactor<T extends HoodieRecordPayload, I, K, O> im
result = compactionHandler.handleInsert(instantTime, operation.getPartitionPath(), operation.getFileId(),
scanner.getRecords());
}
scanner.close();
Iterable<List<WriteStatus>> resultIterable = () -> result;
return StreamSupport.stream(resultIterable.spliterator(), false).flatMap(Collection::stream).peek(s -> {
s.getStat().setTotalUpdatedRecordsCompacted(scanner.getNumMergedRecordsInLog());
@@ -212,7 +214,6 @@ public abstract class HoodieCompactor<T extends HoodieRecordPayload, I, K, O> im
RuntimeStats runtimeStats = new RuntimeStats();
runtimeStats.setTotalScanTime(scanner.getTotalTimeTakenToReadAndMergeBlocks());
s.getStat().setRuntimeStats(runtimeStats);
scanner.close();
}).collect(toList());
}