1
0

refactor classes to accept Map passed by RealtimeCompactor to avoid multiple map creations in HoodieMergeHandle

This commit is contained in:
Nishith Agarwal
2018-01-24 13:34:14 -08:00
committed by vinoth chandar
parent 30049383f5
commit 7076c2e9f0
4 changed files with 100 additions and 60 deletions

View File

@@ -43,6 +43,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -62,8 +63,8 @@ public class HoodieCompactedLogRecordScanner implements
private final static Logger log = LogManager.getLogger(HoodieCompactedLogRecordScanner.class);
// Final list of compacted/merged records to iterate
private final Collection<HoodieRecord<? extends HoodieRecordPayload>> logRecords;
// Final map of compacted/merged records
private final Map<String, HoodieRecord<? extends HoodieRecordPayload>> records;
// Reader schema for the records
private final Schema readerSchema;
// Total log files read - for metrics
@@ -89,7 +90,7 @@ public class HoodieCompactedLogRecordScanner implements
this.payloadClassFQN = this.hoodieTableMetaClient.getTableConfig().getPayloadClass();
// Store merged records for all versions for this log file
Map<String, HoodieRecord<? extends HoodieRecordPayload>> records = Maps.newHashMap();
this.records = Maps.newHashMap();
// iterate over the paths
Iterator<String> logFilePathsItr = logFilePaths.iterator();
while (logFilePathsItr.hasNext()) {
@@ -202,7 +203,6 @@ public class HoodieCompactedLogRecordScanner implements
merge(records, currentInstantLogBlocks);
}
}
this.logRecords = Collections.unmodifiableCollection(records.values());
this.totalRecordsToUpdate = records.size();
}
@@ -297,7 +297,7 @@ public class HoodieCompactedLogRecordScanner implements
@Override
public Iterator<HoodieRecord<? extends HoodieRecordPayload>> iterator() {
return logRecords.iterator();
return records.values().iterator();
}
public long getTotalLogFiles() {
@@ -308,6 +308,10 @@ public class HoodieCompactedLogRecordScanner implements
return totalLogRecords.get();
}
public Map<String, HoodieRecord<? extends HoodieRecordPayload>> getRecords() {
return records;
}
public long getTotalRecordsToUpdate() {
return totalRecordsToUpdate;
}