[HUDI-4276] Reconcile schema-inject null values for missing fields and add new fields (#6017)
* [HUDI-4276] Reconcile schema-inject null values for missing fields and add new fields. * fix comments Co-authored-by: public (bdcee5037027) <mengtao0326@qq.com>
This commit is contained in:
xiarixiaoyao
@@ -27,12 +27,14 @@ import org.apache.avro.Schema;
|
||||
import org.apache.avro.generic.GenericData;
|
||||
import org.apache.avro.generic.GenericRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -98,6 +100,12 @@ public class TestHoodieAvroUtils {
|
||||
+ "{\"name\":\"student\",\"type\":{\"name\":\"student\",\"type\":\"record\",\"fields\":["
|
||||
+ "{\"name\":\"firstname\",\"type\":[\"null\" ,\"string\"],\"default\": null},{\"name\":\"lastname\",\"type\":[\"null\" ,\"string\"],\"default\": null}]}}]}";
|
||||
|
||||
private static String SCHEMA_WITH_NESTED_FIELD_RENAMED = "{\"name\":\"MyClass\",\"type\":\"record\",\"namespace\":\"com.acme.avro\",\"fields\":["
|
||||
+ "{\"name\":\"fn\",\"type\":\"string\"},"
|
||||
+ "{\"name\":\"ln\",\"type\":\"string\"},"
|
||||
+ "{\"name\":\"ss\",\"type\":{\"name\":\"ss\",\"type\":\"record\",\"fields\":["
|
||||
+ "{\"name\":\"fn\",\"type\":[\"null\" ,\"string\"],\"default\": null},{\"name\":\"ln\",\"type\":[\"null\" ,\"string\"],\"default\": null}]}}]}";
|
||||
|
||||
@Test
|
||||
public void testPropsPresent() {
|
||||
Schema schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(EXAMPLE_SCHEMA));
|
||||
@@ -342,4 +350,26 @@ public class TestHoodieAvroUtils {
|
||||
assertEquals(Schema.create(Schema.Type.STRING), getNestedFieldSchemaFromWriteSchema(rec3.getSchema(), "student.firstname"));
|
||||
assertEquals(Schema.create(Schema.Type.STRING), getNestedFieldSchemaFromWriteSchema(nestedSchema, "student.firstname"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReWriteAvroRecordWithNewSchema() {
|
||||
Schema nestedSchema = new Schema.Parser().parse(SCHEMA_WITH_NESTED_FIELD);
|
||||
GenericRecord rec3 = new GenericData.Record(nestedSchema);
|
||||
rec3.put("firstname", "person1");
|
||||
rec3.put("lastname", "person2");
|
||||
GenericRecord studentRecord = new GenericData.Record(rec3.getSchema().getField("student").schema());
|
||||
studentRecord.put("firstname", "person1");
|
||||
studentRecord.put("lastname", "person2");
|
||||
rec3.put("student", studentRecord);
|
||||
|
||||
Schema nestedSchemaRename = new Schema.Parser().parse(SCHEMA_WITH_NESTED_FIELD_RENAMED);
|
||||
Map<String, String> colRenames = new HashMap<>();
|
||||
colRenames.put("fn", "firstname");
|
||||
colRenames.put("ln", "lastname");
|
||||
colRenames.put("ss", "student");
|
||||
colRenames.put("ss.fn", "firstname");
|
||||
colRenames.put("ss.ln", "lastname");
|
||||
GenericRecord studentRecordRename = HoodieAvroUtils.rewriteRecordWithNewSchema(rec3, nestedSchemaRename, colRenames);
|
||||
Assertions.assertEquals(GenericData.get().validate(nestedSchemaRename, studentRecordRename), true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ import java.math.BigDecimal;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -45,6 +46,17 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class TestAvroSchemaEvolutionUtils {
|
||||
|
||||
String schemaStr = "{\"type\":\"record\",\"name\":\"newTableName\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"data\","
|
||||
+ "\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"preferences\",\"type\":[\"null\","
|
||||
+ "{\"type\":\"record\",\"name\":\"newTableName_preferences\",\"fields\":[{\"name\":\"feature1\","
|
||||
+ "\"type\":\"boolean\"},{\"name\":\"feature2\",\"type\":[\"null\",\"boolean\"],\"default\":null}]}],"
|
||||
+ "\"default\":null},{\"name\":\"locations\",\"type\":{\"type\":\"map\",\"values\":{\"type\":\"record\","
|
||||
+ "\"name\":\"newTableName_locations\",\"fields\":[{\"name\":\"lat\",\"type\":\"float\"},{\"name\":\"long\","
|
||||
+ "\"type\":\"float\"}]}}},{\"name\":\"points\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\","
|
||||
+ "{\"type\":\"record\",\"name\":\"newTableName_points\",\"fields\":[{\"name\":\"x\",\"type\":\"long\"},"
|
||||
+ "{\"name\":\"y\",\"type\":\"long\"}]}]}],\"default\":null},{\"name\":\"doubles\",\"type\":{\"type\":\"array\",\"items\":\"double\"}},"
|
||||
+ "{\"name\":\"properties\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"string\"]}],\"default\":null}]}";
|
||||
|
||||
@Test
|
||||
public void testPrimitiveTypes() {
|
||||
Schema[] avroPrimitives = new Schema[] {
|
||||
@@ -146,16 +158,6 @@ public class TestAvroSchemaEvolutionUtils {
|
||||
|
||||
@Test
|
||||
public void testComplexConvert() {
|
||||
String schemaStr = "{\"type\":\"record\",\"name\":\"newTableName\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"data\","
|
||||
+ "\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"preferences\",\"type\":[\"null\","
|
||||
+ "{\"type\":\"record\",\"name\":\"newTableName_preferences\",\"fields\":[{\"name\":\"feature1\","
|
||||
+ "\"type\":\"boolean\"},{\"name\":\"feature2\",\"type\":[\"null\",\"boolean\"],\"default\":null}]}],"
|
||||
+ "\"default\":null},{\"name\":\"locations\",\"type\":{\"type\":\"map\",\"values\":{\"type\":\"record\","
|
||||
+ "\"name\":\"newTableName_locations\",\"fields\":[{\"name\":\"lat\",\"type\":\"float\"},{\"name\":\"long\","
|
||||
+ "\"type\":\"float\"}]}}},{\"name\":\"points\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\","
|
||||
+ "{\"type\":\"record\",\"name\":\"newTableName_points\",\"fields\":[{\"name\":\"x\",\"type\":\"long\"},"
|
||||
+ "{\"name\":\"y\",\"type\":\"long\"}]}]}],\"default\":null},{\"name\":\"doubles\",\"type\":{\"type\":\"array\",\"items\":\"double\"}},"
|
||||
+ "{\"name\":\"properties\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"string\"]}],\"default\":null}]}";
|
||||
Schema schema = new Schema.Parser().parse(schemaStr);
|
||||
|
||||
InternalSchema internalSchema = new InternalSchema(Types.Field.get(0, false, "id", Types.IntType.get()),
|
||||
@@ -284,7 +286,7 @@ public class TestAvroSchemaEvolutionUtils {
|
||||
.updateColumnType("col6", Types.StringType.get());
|
||||
InternalSchema newSchema = SchemaChangeUtils.applyTableChanges2Schema(internalSchema, updateChange);
|
||||
Schema newAvroSchema = AvroInternalSchemaConverter.convert(newSchema, avroSchema.getName());
|
||||
GenericRecord newRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newAvroSchema, new HashMap<>());
|
||||
GenericRecord newRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newAvroSchema, Collections.emptyMap());
|
||||
|
||||
Assertions.assertEquals(GenericData.get().validate(newAvroSchema, newRecord), true);
|
||||
}
|
||||
@@ -349,9 +351,26 @@ public class TestAvroSchemaEvolutionUtils {
|
||||
);
|
||||
|
||||
Schema newAvroSchema = AvroInternalSchemaConverter.convert(newRecord, schema.getName());
|
||||
GenericRecord newAvroRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newAvroSchema, new HashMap<>());
|
||||
GenericRecord newAvroRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newAvroSchema, Collections.emptyMap());
|
||||
// test the correctly of rewrite
|
||||
Assertions.assertEquals(GenericData.get().validate(newAvroSchema, newAvroRecord), true);
|
||||
|
||||
// test rewrite with rename
|
||||
InternalSchema internalSchema = AvroInternalSchemaConverter.convert(schema);
|
||||
// do change rename operation
|
||||
TableChanges.ColumnUpdateChange updateChange = TableChanges.ColumnUpdateChange.get(internalSchema);
|
||||
updateChange
|
||||
.renameColumn("id", "idx")
|
||||
.renameColumn("data", "datax")
|
||||
.renameColumn("preferences.feature1", "f1")
|
||||
.renameColumn("preferences.feature2", "f2")
|
||||
.renameColumn("locations.value.lat", "lt");
|
||||
InternalSchema internalSchemaRename = SchemaChangeUtils.applyTableChanges2Schema(internalSchema, updateChange);
|
||||
Schema avroSchemaRename = AvroInternalSchemaConverter.convert(internalSchemaRename, schema.getName());
|
||||
Map<String, String> renameCols = InternalSchemaUtils.collectRenameCols(internalSchema, internalSchemaRename);
|
||||
GenericRecord avroRecordRename = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, avroSchemaRename, renameCols);
|
||||
// test the correctly of rewrite
|
||||
Assertions.assertEquals(GenericData.get().validate(avroSchemaRename, avroRecordRename), true);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -395,7 +414,7 @@ public class TestAvroSchemaEvolutionUtils {
|
||||
);
|
||||
evolvedRecord = (Types.RecordType)InternalSchemaBuilder.getBuilder().refreshNewId(evolvedRecord, new AtomicInteger(0));
|
||||
Schema evolvedAvroSchema = AvroInternalSchemaConverter.convert(evolvedRecord, "test1");
|
||||
InternalSchema result = AvroSchemaEvolutionUtils.evolveSchemaFromNewAvroSchema(evolvedAvroSchema, oldSchema);
|
||||
InternalSchema result = AvroSchemaEvolutionUtils.reconcileSchema(evolvedAvroSchema, oldSchema);
|
||||
Types.RecordType checkedRecord = Types.RecordType.get(
|
||||
Types.Field.get(0, false, "id", Types.IntType.get()),
|
||||
Types.Field.get(1, true, "data", Types.StringType.get()),
|
||||
@@ -419,4 +438,37 @@ public class TestAvroSchemaEvolutionUtils {
|
||||
);
|
||||
Assertions.assertEquals(result.getRecord(), checkedRecord);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReconcileSchema() {
|
||||
// simple schema test
|
||||
// a: boolean, b: int, c: long, d: date
|
||||
Schema schema = create("simple",
|
||||
new Schema.Field("a", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.BOOLEAN)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("b", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.INT)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("c", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.LONG)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("d", AvroInternalSchemaConverter.nullableSchema(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))), null, JsonProperties.NULL_VALUE));
|
||||
// a: boolean, c: long, c_1: long, d: date
|
||||
Schema incomingSchema = create("simpleIncoming",
|
||||
new Schema.Field("a", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.BOOLEAN)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("a1", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.LONG)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("c", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.LONG)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("c1", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.LONG)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("c2", AvroInternalSchemaConverter.nullableSchema(Schema.create(Schema.Type.LONG)), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("d", AvroInternalSchemaConverter.nullableSchema(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("d1", AvroInternalSchemaConverter.nullableSchema(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))), null, JsonProperties.NULL_VALUE),
|
||||
new Schema.Field("d2", AvroInternalSchemaConverter.nullableSchema(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))), null, JsonProperties.NULL_VALUE));
|
||||
|
||||
Schema simpleCheckSchema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"simpleReconcileSchema\",\"fields\":[{\"name\":\"a\",\"type\":[\"null\",\"boolean\"],\"default\":null},"
|
||||
+ "{\"name\":\"b\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"a1\",\"type\":[\"null\",\"long\"],\"default\":null},"
|
||||
+ "{\"name\":\"c\",\"type\":[\"null\",\"long\"],\"default\":null},"
|
||||
+ "{\"name\":\"c1\",\"type\":[\"null\",\"long\"],\"default\":null},{\"name\":\"c2\",\"type\":[\"null\",\"long\"],\"default\":null},"
|
||||
+ "{\"name\":\"d\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}],\"default\":null},"
|
||||
+ "{\"name\":\"d1\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}],\"default\":null},"
|
||||
+ "{\"name\":\"d2\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}],\"default\":null}]}");
|
||||
|
||||
Schema simpleReconcileSchema = AvroInternalSchemaConverter.convert(AvroSchemaEvolutionUtils
|
||||
.reconcileSchema(incomingSchema, AvroInternalSchemaConverter.convert(schema)), "simpleReconcileSchema");
|
||||
Assertions.assertEquals(simpleReconcileSchema, simpleCheckSchema);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user