lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2170] [HUDI-1763] Always choose the latest record for HoodieRecordPayload (#3401)

Browse Source
This commit is contained in:
swuferhong
2021-08-11 10:20:55 +08:00
committed by GitHub
parent d1b4aa59bf
commit 5448cdde7e
17 changed files with 110 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-common/src/test/java/org/apache/hudi/common/testutils/AvroBinaryTestPayload.java
View File

@@ -44,7 +44,7 @@ public class AvroBinaryTestPayload implements HoodieRecordPayload {
}
@Override
public HoodieRecordPayload preCombine(HoodieRecordPayload another) {
public HoodieRecordPayload preCombine(HoodieRecordPayload oldValue) {
return this;
}

4
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
View File

@@ -233,7 +233,7 @@ public class HoodieTestDataGenerator {
public static RawTripTestPayload generateRandomDeleteValue(HoodieKey key, String instantTime) throws IOException {
GenericRecord rec = generateGenericRecord(key.getRecordKey(), key.getPartitionPath(), "rider-" + instantTime, "driver-" + instantTime, 0,
true, false);
return new RawTripTestPayload(Option.of(rec.toString()), key.getRecordKey(), key.getPartitionPath(), TRIP_EXAMPLE_SCHEMA, true);
return new RawTripTestPayload(Option.of(rec.toString()), key.getRecordKey(), key.getPartitionPath(), TRIP_EXAMPLE_SCHEMA, true, 0L);
}
/**
@@ -574,7 +574,7 @@ public class HoodieTestDataGenerator {
public HoodieRecord generateDeleteRecord(HoodieKey key) throws IOException {
RawTripTestPayload payload =
new RawTripTestPayload(Option.empty(), key.getRecordKey(), key.getPartitionPath(), null, true);
new RawTripTestPayload(Option.empty(), key.getRecordKey(), key.getPartitionPath(), null, true, 0L);
return new HoodieRecord(key, payload);
}

15
hudi-common/src/test/java/org/apache/hudi/common/testutils/RawTripTestPayload.java
View File

@@ -53,9 +53,10 @@ public class RawTripTestPayload implements HoodieRecordPayload<RawTripTestPayloa
private byte[] jsonDataCompressed;
private int dataSize;
private boolean isDeleted;
private Comparable orderingVal;
public RawTripTestPayload(Option<String> jsonData, String rowKey, String partitionPath, String schemaStr,
Boolean isDeleted) throws IOException {
Boolean isDeleted, Comparable orderingVal) throws IOException {
if (jsonData.isPresent()) {
this.jsonDataCompressed = compressData(jsonData.get());
this.dataSize = jsonData.get().length();
@@ -63,10 +64,11 @@ public class RawTripTestPayload implements HoodieRecordPayload<RawTripTestPayloa
this.rowKey = rowKey;
this.partitionPath = partitionPath;
this.isDeleted = isDeleted;
this.orderingVal = orderingVal;
}
public RawTripTestPayload(String jsonData, String rowKey, String partitionPath, String schemaStr) throws IOException {
this(Option.of(jsonData), rowKey, partitionPath, schemaStr, false);
this(Option.of(jsonData), rowKey, partitionPath, schemaStr, false, 0L);
}
public RawTripTestPayload(String jsonData) throws IOException {
@@ -105,8 +107,13 @@ public class RawTripTestPayload implements HoodieRecordPayload<RawTripTestPayloa
}
@Override
public RawTripTestPayload preCombine(RawTripTestPayload another) {
return another;
public RawTripTestPayload preCombine(RawTripTestPayload oldValue) {
if (oldValue.orderingVal.compareTo(orderingVal) > 0) {
// pick the payload with greatest ordering value
return oldValue;
} else {
return this;
}
}
@Override