lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-335] Improvements to DiskBasedMap used by ExternalSpillableMap, for write and random/sequential read paths, by introducing bufferedRandmomAccessFile

Browse Source
This commit is contained in:
Balajee Nagasubramaniam
2020-01-13 12:40:06 -08:00
committed by n3nash
parent 7aa3ce31e6
commit dd09abb56d
6 changed files with 454 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
View File

@@ -311,9 +311,11 @@ public class HoodieMergeHandle<T extends HoodieRecordPayload> extends HoodieWrit
public WriteStatus close() {
try {
// write out any pending records (this can happen when inserts are turned into updates)
for (String key : keyToNewRecords.keySet()) {
if (!writtenRecordKeys.contains(key)) {
HoodieRecord<T> hoodieRecord = keyToNewRecords.get(key);
Iterator<HoodieRecord<T>> newRecordsItr = (keyToNewRecords instanceof ExternalSpillableMap)
? ((ExternalSpillableMap)keyToNewRecords).iterator() : keyToNewRecords.values().iterator();
while (newRecordsItr.hasNext()) {
HoodieRecord<T> hoodieRecord = newRecordsItr.next();
if (!writtenRecordKeys.contains(hoodieRecord.getRecordKey())) {
if (useWriterSchema) {
writeRecord(hoodieRecord, hoodieRecord.getData().getInsertValue(writerSchema));
} else {