lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Reducing memory footprint required in HoodieAvroDataBlock and HoodieAppendHandle

Browse Source
This commit is contained in:
Nishith Agarwal
2017-12-15 14:03:06 -08:00
committed by vinoth chandar
parent 85d32930cd
commit 937ae322ba
8 changed files with 174 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java
View File

@@ -160,14 +160,17 @@ public class HoodieAppendHandle<T extends HoodieRecordPayload> extends HoodieIOH
List<String> keysToDelete = new ArrayList<>();
Map<HoodieLogBlock.LogMetadataType, String> metadata = Maps.newHashMap();
metadata.put(HoodieLogBlock.LogMetadataType.INSTANT_TIME, commitTime);
records.stream().forEach(record -> {
Iterator<HoodieRecord<T>> recordsItr = records.iterator();
while (recordsItr.hasNext()) {
HoodieRecord record = recordsItr.next();
Optional<IndexedRecord> indexedRecord = getIndexedRecord(record);
if (indexedRecord.isPresent()) {
recordList.add(indexedRecord.get());
} else {
keysToDelete.add(record.getRecordKey());
}
});
recordsItr.remove(); //remove entries when IndexedRecord added to new list
}
try {
if (recordList.size() > 0) {
writer = writer.appendBlock(new HoodieAvroDataBlock(recordList, schema, metadata));