lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-89] Add configOption & refactor all configs based on that (#2833)

Browse Source 
Co-authored-by: Wenning Ding <wenningd@amazon.com>
This commit is contained in:
wenningd
2021-06-30 14:26:30 -07:00
committed by GitHub
parent 07e93de8b4
commit d412fb2fe6
173 changed files with 4277 additions and 3309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java
View File

@@ -91,12 +91,10 @@ public class TestBootstrapIndex extends HoodieCommonTestHarness {
@Test
public void testNoOpBootstrapIndex() throws IOException {
Map<String, String> props = metaClient.getTableConfig().getProps();
props.put(HoodieTableConfig.HOODIE_BOOTSTRAP_INDEX_ENABLE, "false");
Properties props = metaClient.getTableConfig().getProps();
props.put(HoodieTableConfig.HOODIE_BOOTSTRAP_INDEX_ENABLE_PROP.key(), "false");
Properties properties = new Properties();
for (Map.Entry<String, String> prop : props.entrySet()) {
properties.setProperty(prop.getKey(), prop.getValue());
}
properties.putAll(props);
HoodieTableConfig.createHoodieProperties(metaClient.getFs(), new Path(metaClient.getMetaPath()), properties);
metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).build();
@@ -187,7 +185,7 @@ public class TestBootstrapIndex extends HoodieCommonTestHarness {
return Arrays.stream(partitions).map(partition -> {
return Pair.of(partition, IntStream.range(0, numEntriesPerPartition).mapToObj(idx -> {
String hudiFileId = UUID.randomUUID().toString();
String sourceFileName = idx + HoodieTableConfig.DEFAULT_BASE_FILE_FORMAT.getFileExtension();
String sourceFileName = idx + HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension();
HoodieFileStatus sourceFileStatus = HoodieFileStatus.newBuilder()
.setPath(HoodiePath.newBuilder().setUri(sourceBasePath + "/" + partition + "/" + sourceFileName).build())
.setLength(256 * 1024 * 1024L)

101
hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigProperty.java Normal file
View File

@@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.common.config;
import org.apache.hudi.common.util.Option;
import org.junit.jupiter.api.Test;
import java.util.Properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestConfigProperty extends HoodieConfig {
public static ConfigProperty<String> FAKE_STRING_CONFIG = ConfigProperty
.key("test.fake.string.config")
.defaultValue("1")
.withAlternatives("test.fake.string.alternative.config")
.withDocumentation("Fake config only for testing");
public static ConfigProperty<String> FAKE_BOOLEAN_CONFIG = ConfigProperty
.key("test.fake.boolean.config")
.defaultValue("false")
.withDocumentation("Fake config only for testing");
public static ConfigProperty<Integer> FAKE_INTEGER_CONFIG = ConfigProperty
.key("test.fake.integer.config")
.defaultValue(0)
.withInferFunction(p -> {
if (p.contains(FAKE_STRING_CONFIG) && p.getString(FAKE_STRING_CONFIG).equals("5")) {
return Option.of(100);
}
return Option.empty();
})
.withDocumentation("Fake config only for testing");
@Test
public void testGetTypedValue() {
HoodieConfig hoodieConfig = new HoodieConfig();
assertNull(hoodieConfig.getInt(FAKE_STRING_CONFIG));
hoodieConfig.setValue(FAKE_STRING_CONFIG, "5");
assertEquals(5, hoodieConfig.getInt(FAKE_STRING_CONFIG));
assertNull(hoodieConfig.getBoolean(FAKE_BOOLEAN_CONFIG));
hoodieConfig.setValue(FAKE_BOOLEAN_CONFIG, "true");
assertEquals(true, hoodieConfig.getBoolean(FAKE_BOOLEAN_CONFIG));
}
@Test
public void testGetOrDefault() {
Properties props = new Properties();
props.put("test.unknown.config", "abc");
HoodieConfig hoodieConfig = new HoodieConfig(props);
assertEquals("1", hoodieConfig.getStringOrDefault(FAKE_STRING_CONFIG));
assertEquals("2", hoodieConfig.getStringOrDefault(FAKE_STRING_CONFIG, "2"));
}
@Test
public void testAlternatives() {
Properties props = new Properties();
props.put("test.fake.string.alternative.config", "1");
HoodieConfig hoodieConfig = new HoodieConfig(props);
assertTrue(hoodieConfig.contains(FAKE_STRING_CONFIG));
assertEquals("1", hoodieConfig.getString(FAKE_STRING_CONFIG));
}
@Test
public void testInference() {
HoodieConfig hoodieConfig1 = new HoodieConfig();
hoodieConfig1.setDefaultValue(FAKE_INTEGER_CONFIG);
assertEquals(0, hoodieConfig1.getInt(FAKE_INTEGER_CONFIG));
HoodieConfig hoodieConfig2 = new HoodieConfig();
hoodieConfig2.setValue(FAKE_STRING_CONFIG, "5");
hoodieConfig2.setDefaultValue(FAKE_INTEGER_CONFIG);
assertEquals(100, hoodieConfig2.getInt(FAKE_INTEGER_CONFIG));
}
@Test
public void testSetDefaults() {
setDefaults(this.getClass().getName());
assertEquals(3, getProps().size());
}
}

2
hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
View File

@@ -57,7 +57,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
private final long minCleanToKeep = 10;
private static String TEST_WRITE_TOKEN = "1-0-1";
private static final String BASE_FILE_EXTENSION = HoodieTableConfig.DEFAULT_BASE_FILE_FORMAT.getFileExtension();
private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension();
@Rule
public final EnvironmentVariables environmentVariables = new EnvironmentVariables();

4
hudi-common/src/test/java/org/apache/hudi/common/model/TestDefaultHoodieRecordPayload.java
View File

@@ -54,8 +54,8 @@ public class TestDefaultHoodieRecordPayload {
new Schema.Field("_hoodie_is_deleted", Schema.create(Type.BOOLEAN), "", false)
));
props = new Properties();
props.setProperty(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP, "ts");
props.setProperty(HoodiePayloadProps.PAYLOAD_EVENT_TIME_FIELD_PROP, "ts");
props.setProperty(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP_KEY, "ts");
props.setProperty(HoodiePayloadProps.PAYLOAD_EVENT_TIME_FIELD_PROP_KEY, "ts");
}
@Test

2
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieDeltaWriteStat.java
View File

@@ -34,7 +34,7 @@ public class TestHoodieDeltaWriteStat {
@Test
public void testBaseFileAndLogFiles() {
HoodieDeltaWriteStat writeStat = new HoodieDeltaWriteStat();
String baseFile = "file1" + HoodieTableConfig.DEFAULT_BASE_FILE_FORMAT.getFileExtension();
String baseFile = "file1" + HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension();
String logFile1 = ".log1.log";
String logFile2 = ".log2.log";

2
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
View File

@@ -68,7 +68,7 @@ public class TestPriorityBasedFileSystemView {
fsView = new PriorityBasedFileSystemView(primary, secondary);
testBaseFileStream = Stream.of(new HoodieBaseFile("test"));
testFileSliceStream = Stream.of(new FileSlice("2020-01-01", "20:20",
"file0001" + HoodieTableConfig.DEFAULT_BASE_FILE_FORMAT.getFileExtension()));
"file0001" + HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension()));
}
private void resetMocks() {

2
hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
View File

@@ -60,7 +60,7 @@ import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serial
public class FileCreateUtils {
private static final String WRITE_TOKEN = "1-0-1";
private static final String BASE_FILE_EXTENSION = HoodieTableConfig.DEFAULT_BASE_FILE_FORMAT.getFileExtension();
private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension();
public static String baseFileName(String instantTime, String fileId) {
return baseFileName(instantTime, fileId, BASE_FILE_EXTENSION);

2
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
View File

@@ -397,7 +397,7 @@ public class HoodieTestTable {
}
public FileStatus[] listAllBaseFiles() throws IOException {
return listAllBaseFiles(HoodieTableConfig.DEFAULT_BASE_FILE_FORMAT.getFileExtension());
return listAllBaseFiles(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension());
}
public FileStatus[] listAllBaseFiles(String fileExtension) throws IOException {

6
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
View File

@@ -65,7 +65,7 @@ public class HoodieTestUtils {
public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath) throws IOException {
Properties props = new Properties();
props.setProperty(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH, bootstrapBasePath);
props.setProperty(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP.key(), bootstrapBasePath);
return init(getDefaultHadoopConf(), basePath, tableType, props);
}
@@ -86,7 +86,7 @@ public class HoodieTestUtils {
String tableName)
throws IOException {
Properties properties = new Properties();
properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tableName);
properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP.key(), tableName);
return init(hadoopConf, basePath, tableType, properties);
}
@@ -94,7 +94,7 @@ public class HoodieTestUtils {
HoodieFileFormat baseFileFormat)
throws IOException {
Properties properties = new Properties();
properties.setProperty(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP_NAME, baseFileFormat.toString());
properties.setProperty(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.key(), baseFileFormat.toString());
return init(hadoopConf, basePath, tableType, properties);
}

2
hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
View File

@@ -37,7 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public final class TestTablePathUtils {
private static final String BASE_FILE_EXTENSION = HoodieTableConfig.DEFAULT_BASE_FILE_FORMAT.getFileExtension();
private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension();
@TempDir
static File tempDir;