lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Fix EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION config (#3250)

Browse Source
This commit is contained in:
Sagar Sumit
2021-07-13 09:54:40 +05:30
committed by GitHub
parent c8a2033c27
commit b0089b894a
2 changed files with 14 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -339,8 +339,9 @@ public class HoodieWriteConfig extends HoodieConfig {
.withDocumentation(""); .withDocumentation("");
public static final ConfigProperty<String> EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION = ConfigProperty public static final ConfigProperty<String> EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION = ConfigProperty
.key(AVRO_SCHEMA + ".externalTransformation") .key(AVRO_SCHEMA.key() + ".external.transformation")
.defaultValue("false") .defaultValue("false")
.withAlternatives(AVRO_SCHEMA.key() + ".externalTransformation")
.withDocumentation(""); .withDocumentation("");
private ConsistencyGuardConfig consistencyGuardConfig; private ConsistencyGuardConfig consistencyGuardConfig;

14
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
View File

@@ -23,6 +23,8 @@ import org.apache.hudi.config.HoodieWriteConfig.Builder;
import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@@ -33,16 +35,23 @@ import java.util.Map;
import java.util.Properties; import java.util.Properties;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieWriteConfig { public class TestHoodieWriteConfig {
@Test @ParameterizedTest
public void testPropertyLoading() throws IOException { @ValueSource(booleans = {true, false})
public void testPropertyLoading(boolean withAlternative) throws IOException {
Builder builder = HoodieWriteConfig.newBuilder().withPath("/tmp"); Builder builder = HoodieWriteConfig.newBuilder().withPath("/tmp");
Map<String, String> params = new HashMap<>(3); Map<String, String> params = new HashMap<>(3);
params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP.key(), "1"); params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP.key(), "1");
params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP.key(), "5"); params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP.key(), "5");
params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP.key(), "2"); params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP.key(), "2");
if (withAlternative) {
params.put("hoodie.avro.schema.externalTransformation", "true");
} else {
params.put("hoodie.avro.schema.external.transformation", "true");
}
ByteArrayOutputStream outStream = saveParamsIntoOutputStream(params); ByteArrayOutputStream outStream = saveParamsIntoOutputStream(params);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outStream.toByteArray()); ByteArrayInputStream inputStream = new ByteArrayInputStream(outStream.toByteArray());
try { try {
@@ -54,6 +63,7 @@ public class TestHoodieWriteConfig {
HoodieWriteConfig config = builder.build(); HoodieWriteConfig config = builder.build();
assertEquals(5, config.getMaxCommitsToKeep()); assertEquals(5, config.getMaxCommitsToKeep());
assertEquals(2, config.getMinCommitsToKeep()); assertEquals(2, config.getMinCommitsToKeep());
assertTrue(config.shouldUseExternalSchemaTransformation());
} }
@Test @Test