lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1058] Make delete marker configurable (#1819)

Browse Source
This commit is contained in:
Shen Hong
2020-08-03 23:06:31 +08:00
committed by GitHub
parent 8aa9142de8
commit 433d7d2c98
13 changed files with 264 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteWithLatestAvroPayload.java
View File

@@ -36,6 +36,8 @@ import java.io.IOException;
public class OverwriteWithLatestAvroPayload extends BaseAvroPayload
implements HoodieRecordPayload<OverwriteWithLatestAvroPayload> {
private String deleteMarkerField = "_hoodie_is_deleted";
/**
*
*/
@@ -47,6 +49,12 @@ public class OverwriteWithLatestAvroPayload extends BaseAvroPayload
this(record.isPresent() ? record.get() : null, (record1) -> 0); // natural order
}
public OverwriteWithLatestAvroPayload(GenericRecord record, Comparable orderingVal,
String deleteMarkerField) {
this(record, orderingVal);
this.deleteMarkerField = deleteMarkerField;
}
@Override
public OverwriteWithLatestAvroPayload preCombine(OverwriteWithLatestAvroPayload another) {
// pick the payload with greatest ordering value
@@ -80,7 +88,7 @@ public class OverwriteWithLatestAvroPayload extends BaseAvroPayload
* @returns {@code true} if record represents a delete record. {@code false} otherwise.
*/
private boolean isDeleteRecord(GenericRecord genericRecord) {
Object deleteMarker = genericRecord.get("_hoodie_is_deleted");
Object deleteMarker = genericRecord.get(deleteMarkerField);
return (deleteMarker instanceof Boolean && (boolean) deleteMarker);
}
}