lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1058] Make delete marker configurable (#1819)

Browse Source
This commit is contained in:
Shen Hong
2020-08-03 23:06:31 +08:00
committed by GitHub
parent 8aa9142de8
commit 433d7d2c98
13 changed files with 264 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteWithLatestAvroPayload.java
View File

@@ -37,6 +37,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
public class TestOverwriteWithLatestAvroPayload {
private Schema schema;
String defaultDeleteMarkerField = "_hoodie_is_deleted";
String deleteMarkerField = "delete_marker_field";
@BeforeEach
public void setUp() throws Exception {
@@ -44,26 +46,56 @@ public class TestOverwriteWithLatestAvroPayload {
new Schema.Field("id", Schema.create(Schema.Type.STRING), "", null),
new Schema.Field("partition", Schema.create(Schema.Type.STRING), "", null),
new Schema.Field("ts", Schema.create(Schema.Type.LONG), "", null),
new Schema.Field("_hoodie_is_deleted", Schema.create(Type.BOOLEAN), "", false)
new Schema.Field(defaultDeleteMarkerField, Schema.create(Type.BOOLEAN), "", false),
new Schema.Field(deleteMarkerField, Schema.create(Type.BOOLEAN), "", false)
));
}
@Test
public void testActiveRecords() throws IOException {
public void testOverwriteWithLatestAvroPayload() throws IOException {
GenericRecord record1 = new GenericData.Record(schema);
record1.put("id", "1");
record1.put("partition", "partition0");
record1.put("ts", 0L);
record1.put("_hoodie_is_deleted", false);
record1.put(defaultDeleteMarkerField, false);
record1.put(deleteMarkerField, false);
// test1: set default marker field value to true and user defined to false
GenericRecord record2 = new GenericData.Record(schema);
record2.put("id", "2");
record2.put("partition", "partition1");
record2.put("ts", 1L);
record2.put("_hoodie_is_deleted", false);
record2.put(defaultDeleteMarkerField, true);
record2.put(deleteMarkerField, false);
// set to user defined marker field with false, the record should be considered active.
assertActiveRecord(record1, record2, deleteMarkerField);
// set to default marker field with true, the record should be considered delete.
assertDeletedRecord(record1, record2, defaultDeleteMarkerField);
// test2: set default marker field value to false and user defined to true
GenericRecord record3 = new GenericData.Record(schema);
record3.put("id", "2");
record3.put("partition", "partition1");
record3.put("ts", 1L);
record3.put(defaultDeleteMarkerField, false);
record3.put(deleteMarkerField, true);
// set to user defined marker field with true, the record should be considered delete.
assertDeletedRecord(record1, record3, deleteMarkerField);
// set to default marker field with false, the record should be considered active.
assertActiveRecord(record1, record3, defaultDeleteMarkerField);
}
private void assertActiveRecord(GenericRecord record1,
GenericRecord record2, String field) throws IOException {
OverwriteWithLatestAvroPayload payload1 = new OverwriteWithLatestAvroPayload(
record1, 1, field);
OverwriteWithLatestAvroPayload payload2 = new OverwriteWithLatestAvroPayload(
record2, 2, field);
OverwriteWithLatestAvroPayload payload1 = new OverwriteWithLatestAvroPayload(record1, 1);
OverwriteWithLatestAvroPayload payload2 = new OverwriteWithLatestAvroPayload(record2, 2);
assertEquals(payload1.preCombine(payload2), payload2);
assertEquals(payload2.preCombine(payload1), payload2);
@@ -74,22 +106,12 @@ public class TestOverwriteWithLatestAvroPayload {
assertEquals(payload2.combineAndGetUpdateValue(record1, schema).get(), record2);
}
@Test
public void testDeletedRecord() throws IOException {
GenericRecord record1 = new GenericData.Record(schema);
record1.put("id", "1");
record1.put("partition", "partition0");
record1.put("ts", 0L);
record1.put("_hoodie_is_deleted", false);
GenericRecord delRecord1 = new GenericData.Record(schema);
delRecord1.put("id", "2");
delRecord1.put("partition", "partition1");
delRecord1.put("ts", 1L);
delRecord1.put("_hoodie_is_deleted", true);
OverwriteWithLatestAvroPayload payload1 = new OverwriteWithLatestAvroPayload(record1, 1);
OverwriteWithLatestAvroPayload payload2 = new OverwriteWithLatestAvroPayload(delRecord1, 2);
private void assertDeletedRecord(GenericRecord record1,
GenericRecord delRecord1, String field) throws IOException {
OverwriteWithLatestAvroPayload payload1 = new OverwriteWithLatestAvroPayload(
record1, 1, field);
OverwriteWithLatestAvroPayload payload2 = new OverwriteWithLatestAvroPayload(
delRecord1, 2, field);
assertEquals(payload1.preCombine(payload2), payload2);
assertEquals(payload2.preCombine(payload1), payload2);
@@ -99,5 +121,4 @@ public class TestOverwriteWithLatestAvroPayload {
assertEquals(payload1.combineAndGetUpdateValue(delRecord1, schema).get(), record1);
assertFalse(payload2.combineAndGetUpdateValue(record1, schema).isPresent());
}
}

27
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
View File

@@ -105,6 +105,7 @@ public class HoodieTestDataGenerator {
+ "{\"name\": \"seconds_since_epoch\", \"type\": \"long\"},"
+ "{\"name\": \"weight\", \"type\": \"float\"},"
+ "{\"name\": \"nation\", \"type\": \"bytes\"},"
+ "{\"name\": \"user_defined_delete_marker_field\", \"type\": \"boolean\", \"default\": false},"
+ "{\"name\":\"current_date\",\"type\": {\"type\": \"int\", \"logicalType\": \"date\"}},"
+ "{\"name\":\"current_ts\",\"type\": {\"type\": \"long\", \"logicalType\": \"timestamp-micros\"}},"
+ "{\"name\":\"height\",\"type\":{\"type\":\"fixed\",\"name\":\"abc\",\"size\":5,\"logicalType\":\"decimal\",\"precision\":10,\"scale\":6}},";
@@ -122,7 +123,7 @@ public class HoodieTestDataGenerator {
+ "{\"name\":\"driver\",\"type\":\"string\"},{\"name\":\"fare\",\"type\":\"double\"},{\"name\": \"_hoodie_is_deleted\", \"type\": \"boolean\", \"default\": false}]}";
public static final String NULL_SCHEMA = Schema.create(Schema.Type.NULL).toString();
public static final String TRIP_HIVE_COLUMN_TYPES = "double,string,string,string,double,double,double,double,int,bigint,float,binary,int,bigint,decimal(10,6),"
public static final String TRIP_HIVE_COLUMN_TYPES = "double,string,string,string,double,double,double,double,int,bigint,float,binary,boolean,int,bigint,decimal(10,6),"
+ "map<string,string>,struct<amount:double,currency:string>,array<struct<amount:double,currency:string>>,boolean";
public static final Schema AVRO_SCHEMA = new Schema.Parser().parse(TRIP_EXAMPLE_SCHEMA);
@@ -177,6 +178,18 @@ public class HoodieTestDataGenerator {
return null;
}
public static List<GenericRecord> generateGenericRecords(int n, boolean isDeleteRecord, int instantTime) {
return IntStream.range(0, n).boxed().map(i -> {
String partitionPath = DEFAULT_FIRST_PARTITION_PATH;
HoodieKey key = new HoodieKey("id_" + i, partitionPath);
HoodieTestDataGenerator.KeyPartition kp = new HoodieTestDataGenerator.KeyPartition();
kp.key = key;
kp.partitionPath = partitionPath;
return HoodieTestDataGenerator.generateGenericRecord(
key.getRecordKey(), "rider-" + instantTime, "driver-" + instantTime, instantTime, isDeleteRecord, false);
}).collect(Collectors.toList());
}
/**
* Generates a new avro record of the above nested schema format,
* retaining the key if optionally provided.
@@ -263,11 +276,11 @@ public class HoodieTestDataGenerator {
rec.put("weight", RAND.nextFloat());
byte[] bytes = "Canada".getBytes();
rec.put("nation", ByteBuffer.wrap(bytes));
rec.put("user_defined_delete_marker_field", isDeleteRecord);
long currentTimeMillis = System.currentTimeMillis();
Date date = new Date(currentTimeMillis);
rec.put("current_date", (int) date.toLocalDate().toEpochDay());
rec.put("current_ts", currentTimeMillis);
BigDecimal bigDecimal = new BigDecimal(String.format("%5f", RAND.nextFloat()));
Schema decimalSchema = AVRO_SCHEMA.getField("height").schema();
Conversions.DecimalConversion decimalConversions = new Conversions.DecimalConversion();
@@ -290,11 +303,7 @@ public class HoodieTestDataGenerator {
rec.put("tip_history", tipHistoryArray);
}
if (isDeleteRecord) {
rec.put("_hoodie_is_deleted", true);
} else {
rec.put("_hoodie_is_deleted", false);
}
rec.put("_hoodie_is_deleted", isDeleteRecord);
return rec;
}
@@ -761,8 +770,8 @@ public class HoodieTestDataGenerator {
public static class KeyPartition implements Serializable {
HoodieKey key;
String partitionPath;
public HoodieKey key;
public String partitionPath;
}
public void close() {