lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Handling duplicate record update for single partition (duplicates in single or different parquet files)

Browse Source
This commit is contained in:
Omkar Joshi
2019-03-01 15:34:46 -08:00
committed by vinoth chandar
parent b514e1ab18
commit 4a8bec7ea5
3 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hoodie-client/src/main/java/com/uber/hoodie/io/HoodieMergeHandle.java
View File

@@ -202,9 +202,11 @@ public class HoodieMergeHandle<T extends HoodieRecordPayload> extends HoodieIOHa
*/
public void write(GenericRecord oldRecord) {
String key = oldRecord.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
HoodieRecord<T> hoodieRecord = keyToNewRecords.get(key);
boolean copyOldRecord = true;
if (keyToNewRecords.containsKey(key)) {
// If we have duplicate records that we are updating, then the hoodie record will be deflated after
// writing the first record. So make a copy of the record to be merged
HoodieRecord<T> hoodieRecord = new HoodieRecord<>(keyToNewRecords.get(key));
try {
Optional<IndexedRecord> combinedAvroRecord = hoodieRecord.getData()
.combineAndGetUpdateValue(oldRecord, schema);