From 0364498ae3be553f7c9fab78469283a6958da2ff Mon Sep 17 00:00:00 2001 From: wenningd Date: Thu, 5 Nov 2020 17:30:17 -0800 Subject: [PATCH] [HUDI-1375] Fix bug in HoodieAvroUtils.removeMetadataFields() method (#2232) Co-authored-by: Wenning Ding --- .../java/org/apache/hudi/avro/HoodieAvroUtils.java | 1 + .../java/org/apache/hudi/avro/TestHoodieAvroUtils.java | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java index a7517a40d..3b356a792 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java @@ -204,6 +204,7 @@ public class HoodieAvroUtils { List filteredFields = schema.getFields() .stream() .filter(field -> !HoodieRecord.HOODIE_META_COLUMNS.contains(field.name())) + .map(field -> new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal())) .collect(Collectors.toList()); Schema filteredSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), false); filteredSchema.setFields(filteredFields); diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java index f6854ca2b..40db67b50 100644 --- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java @@ -55,6 +55,8 @@ public class TestHoodieAvroUtils { + "{\"name\": \"non_pii_col\", \"type\": \"string\"}," + "{\"name\": \"pii_col\", \"type\": \"string\", \"column_category\": \"user_profile\"}]}"; + private static int NUM_FIELDS_IN_EXAMPLE_SCHEMA = 4; + private static String SCHEMA_WITH_METADATA_FIELD = "{\"type\": \"record\",\"name\": \"testrec2\",\"fields\": [ " + "{\"name\": \"timestamp\",\"type\": \"double\"},{\"name\": \"_row_key\", \"type\": \"string\"}," + "{\"name\": \"non_pii_col\", \"type\": \"string\"}," @@ -197,4 +199,12 @@ public class TestHoodieAvroUtils { //evolvedField5.defaultVal() returns null. assertNull(rec1.get("evolved_field1")); } + + @Test + public void testAddingAndRemovingMetadataFields() { + Schema schemaWithMetaCols = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(EXAMPLE_SCHEMA)); + assertEquals(schemaWithMetaCols.getFields().size(), NUM_FIELDS_IN_EXAMPLE_SCHEMA + HoodieRecord.HOODIE_META_COLUMNS.size()); + Schema schemaWithoutMetaCols = HoodieAvroUtils.removeMetadataFields(schemaWithMetaCols); + assertEquals(schemaWithoutMetaCols.getFields().size(), NUM_FIELDS_IN_EXAMPLE_SCHEMA); + } }