lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Fixing missing hoodie record location in HoodieRecord when record is read from disk after being spilled

Browse Source
This commit is contained in:
Nishith Agarwal
2018-07-12 17:45:10 -07:00
committed by vinoth chandar
parent f62890ca1f
commit 44caf0d40c
5 changed files with 44 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hoodie-common/src/test/java/com/uber/hoodie/common/util/SpillableMapTestUtils.java
View File

@@ -19,6 +19,7 @@ package com.uber.hoodie.common.util;
import com.uber.hoodie.common.model.HoodieAvroPayload;
import com.uber.hoodie.common.model.HoodieKey;
import com.uber.hoodie.common.model.HoodieRecord;
import com.uber.hoodie.common.model.HoodieRecordLocation;
import com.uber.hoodie.common.model.HoodieRecordPayload;
import java.util.ArrayList;
import java.util.List;
@@ -29,6 +30,9 @@ import org.apache.avro.generic.IndexedRecord;
public class SpillableMapTestUtils {
public static final String DUMMY_COMMIT_TIME = "DUMMY_COMMIT_TIME";
public static final String DUMMY_FILE_ID = "DUMMY_FILE_ID";
public static List<String> upsertRecords(List<IndexedRecord> iRecords,
Map<String, HoodieRecord<? extends HoodieRecordPayload>> records) {
List<String> recordKeys = new ArrayList<>();
@@ -38,8 +42,10 @@ public class SpillableMapTestUtils {
String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString();
recordKeys.add(key);
records.put(key, new HoodieRecord<>(new HoodieKey(key, partitionPath),
new HoodieAvroPayload(Optional.of((GenericRecord) r))));
HoodieRecord record = new HoodieRecord<>(new HoodieKey(key, partitionPath),
new HoodieAvroPayload(Optional.of((GenericRecord) r)));
record.setCurrentLocation(new HoodieRecordLocation("DUMMY_COMMIT_TIME", "DUMMY_FILE_ID"));
records.put(key, record);
});
return recordKeys;
}

7
hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestExternalSpillableMap.java
View File

@@ -149,11 +149,12 @@ public class TestExternalSpillableMap {
new HoodieAvroPayload(Optional.of((GenericRecord) onDiskRecord)));
// assert size
assert records.size() == 100;
// get should return the same HoodieKey and same value
// get should return the same HoodieKey, same location and same value
assert inMemoryHoodieRecord.getKey().equals(records.get(ikey).getKey());
assert onDiskHoodieRecord.getKey().equals(records.get(dkey).getKey());
//assert inMemoryHoodieRecord.equals(records.get(ikey));
//assert onDiskHoodieRecord.equals(records.get(dkey));
// compare the member variables of HoodieRecord not set by the constructor
assert records.get(ikey).getCurrentLocation().getFileId().equals(SpillableMapTestUtils.DUMMY_FILE_ID);
assert records.get(ikey).getCurrentLocation().getCommitTime().equals(SpillableMapTestUtils.DUMMY_COMMIT_TIME);
// test contains
assertTrue(records.containsKey(ikey));