From b0089b894ad12da11fbd6a0fb08508c7adee68e6 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Tue, 13 Jul 2021 09:54:40 +0530 Subject: [PATCH] [MINOR] Fix EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION config (#3250) --- .../org/apache/hudi/config/HoodieWriteConfig.java | 3 ++- .../apache/hudi/config/TestHoodieWriteConfig.java | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 20d2846a7..e2e295d3e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -339,8 +339,9 @@ public class HoodieWriteConfig extends HoodieConfig { .withDocumentation(""); public static final ConfigProperty EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION = ConfigProperty - .key(AVRO_SCHEMA + ".externalTransformation") + .key(AVRO_SCHEMA.key() + ".external.transformation") .defaultValue("false") + .withAlternatives(AVRO_SCHEMA.key() + ".externalTransformation") .withDocumentation(""); private ConsistencyGuardConfig consistencyGuardConfig; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java index 7661e1db4..89f7a9798 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java @@ -23,6 +23,8 @@ import org.apache.hudi.config.HoodieWriteConfig.Builder; import org.apache.hudi.index.HoodieIndex; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -33,16 +35,23 @@ import java.util.Map; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHoodieWriteConfig { - @Test - public void testPropertyLoading() throws IOException { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testPropertyLoading(boolean withAlternative) throws IOException { Builder builder = HoodieWriteConfig.newBuilder().withPath("/tmp"); Map params = new HashMap<>(3); params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP.key(), "1"); params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP.key(), "5"); params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP.key(), "2"); + if (withAlternative) { + params.put("hoodie.avro.schema.externalTransformation", "true"); + } else { + params.put("hoodie.avro.schema.external.transformation", "true"); + } ByteArrayOutputStream outStream = saveParamsIntoOutputStream(params); ByteArrayInputStream inputStream = new ByteArrayInputStream(outStream.toByteArray()); try { @@ -54,6 +63,7 @@ public class TestHoodieWriteConfig { HoodieWriteConfig config = builder.build(); assertEquals(5, config.getMaxCommitsToKeep()); assertEquals(2, config.getMinCommitsToKeep()); + assertTrue(config.shouldUseExternalSchemaTransformation()); } @Test