lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3520] Introduce DeleteSupportSchemaPostProcessor to support adding _hoodie_is_deleted column to schema (#4921)

Browse Source
This commit is contained in:
wangxianghu
2022-03-07 00:37:09 +04:00
committed by GitHub
parent 051ad0b033
commit 4b471772aa
5 changed files with 81 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-utilities/src/test/java/org/apache/hudi/utilities/DummySchemaProvider.java
View File

@@ -18,9 +18,10 @@
package org.apache.hudi.utilities;
import org.apache.avro.Schema;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.utilities.schema.SchemaProvider;
import org.apache.avro.Schema;
import org.apache.spark.api.java.JavaSparkContext;
public class DummySchemaProvider extends SchemaProvider {

3
hudi-utilities/src/test/java/org/apache/hudi/utilities/SparkAvroSchemaProvider.java
View File

@@ -18,9 +18,10 @@
package org.apache.hudi.utilities;
import org.apache.avro.Schema;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.utilities.schema.SchemaProvider;
import org.apache.avro.Schema;
import org.apache.spark.api.java.JavaSparkContext;
public class SparkAvroSchemaProvider extends SchemaProvider {

12
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestSchemaPostProcessor.java
View File

@@ -19,17 +19,17 @@
package org.apache.hudi.utilities;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.utilities.schema.DeleteSupportSchemaPostProcessor;
import org.apache.hudi.utilities.schema.SchemaPostProcessor;
import org.apache.hudi.utilities.schema.SchemaPostProcessor.Config;
import org.apache.hudi.utilities.schema.SchemaProvider;
import org.apache.hudi.utilities.schema.SparkAvroPostProcessor;
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
import org.apache.hudi.utilities.transform.FlatteningTransformer;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Type;
import org.apache.avro.SchemaBuilder;
import org.apache.hudi.utilities.transform.FlatteningTransformer;
import org.apache.spark.api.java.JavaSparkContext;
import org.junit.jupiter.api.Test;
@@ -82,6 +82,14 @@ public class TestSchemaPostProcessor extends UtilitiesTestBase {
assertNotNull(schema.getField("day"));
}
@Test
public void testDeleteSupport() {
DeleteSupportSchemaPostProcessor processor = new DeleteSupportSchemaPostProcessor(properties, null);
Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA);
Schema targetSchema = processor.processSchema(schema);
assertNotNull(targetSchema.getField("_hoodie_is_deleted"));
}
@Test
public void testSparkAvroSchema() throws IOException {
SparkAvroPostProcessor processor = new SparkAvroPostProcessor(properties, null);