lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2410] Fix getDefaultBootstrapIndexClass logical error (#3633)

Browse Source
This commit is contained in:
liujinhui
2021-09-13 16:10:17 +08:00
committed by GitHub
parent c79017cb74
commit 9f3c4a2a7f
6 changed files with 31 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
View File

@@ -1252,7 +1252,7 @@ public class TestCleaner extends HoodieClientTestBase {
assertTrue(new File(sourcePath.toString()).exists());
// recreate metaClient with Bootstrap base path
metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString());
metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString(), true);
// generate bootstrap index
Map<String, List<BootstrapFileMapping>> bootstrapMapping = TestBootstrapIndex.generateBootstrapIndex(metaClient, sourcePath.toString(),

9
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
View File

@@ -136,10 +136,10 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable {
.defaultValue("archived")
.withDocumentation("path under the meta folder, to store archived timeline instants at.");
public static final ConfigProperty<String> BOOTSTRAP_INDEX_ENABLE = ConfigProperty
public static final ConfigProperty<Boolean> BOOTSTRAP_INDEX_ENABLE = ConfigProperty
.key("hoodie.bootstrap.index.enable")
.noDefaultValue()
.withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined.");
.defaultValue(true)
.withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined, default true.");
public static final ConfigProperty<String> BOOTSTRAP_INDEX_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.index.class")
@@ -298,8 +298,9 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable {
}
public static String getDefaultBootstrapIndexClass(Properties props) {
HoodieConfig hoodieConfig = new HoodieConfig(props);
String defaultClass = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue();
if ("false".equalsIgnoreCase(props.getProperty(BOOTSTRAP_INDEX_ENABLE.key()))) {
if (!hoodieConfig.getBooleanOrDefault(BOOTSTRAP_INDEX_ENABLE)) {
defaultClass = NO_OP_BOOTSTRAP_INDEX_CLASS;
}
return defaultClass;

15
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
View File

@@ -627,6 +627,7 @@ public class HoodieTableMetaClient implements Serializable {
private String partitionFields;
private String bootstrapIndexClass;
private String bootstrapBasePath;
private Boolean bootstrapIndexEnable;
private Boolean populateMetaFields;
private String keyGeneratorClassProp;
@@ -702,6 +703,11 @@ public class HoodieTableMetaClient implements Serializable {
return this;
}
public PropertyBuilder setBootstrapIndexEnable(Boolean bootstrapIndexEnable) {
this.bootstrapIndexEnable = bootstrapIndexEnable;
return this;
}
public PropertyBuilder setPopulateMetaFields(boolean populateMetaFields) {
this.populateMetaFields = populateMetaFields;
return this;
@@ -749,6 +755,11 @@ public class HoodieTableMetaClient implements Serializable {
if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_BASE_PATH)) {
setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH));
}
if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE)) {
setBootstrapIndexEnable(hoodieConfig.getBoolean(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE));
}
if (hoodieConfig.contains(HoodieTableConfig.PRECOMBINE_FIELD)) {
setPreCombineField(hoodieConfig.getString(HoodieTableConfig.PRECOMBINE_FIELD));
}
@@ -807,6 +818,10 @@ public class HoodieTableMetaClient implements Serializable {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME, bootstrapIndexClass);
}
if (null != bootstrapIndexEnable) {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE, Boolean.toString(bootstrapIndexEnable));
}
if (null != bootstrapBasePath) {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_BASE_PATH, bootstrapBasePath);
}

7
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
View File

@@ -108,7 +108,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
@BeforeEach
public void setup() throws IOException {
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH);
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, false);
basePath = metaClient.getBasePath();
refreshFsView();
}
@@ -344,6 +344,11 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
protected void testViewForFileSlicesWithAsyncCompaction(boolean skipCreatingDataFile, boolean isCompactionInFlight,
int expTotalFileSlices, int expTotalDataFiles, boolean includeInvalidAndInflight, boolean testBootstrap)
throws Exception {
if (testBootstrap) {
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, testBootstrap);
}
String partitionPath = "2016/05/01";
new File(basePath + "/" + partitionPath).mkdirs();
String fileId = UUID.randomUUID().toString();

3
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
View File

@@ -63,9 +63,10 @@ public class HoodieTestUtils {
return init(getDefaultHadoopConf(), basePath, tableType);
}
public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath) throws IOException {
public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath, boolean bootstrapIndexEnable) throws IOException {
Properties props = new Properties();
props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath);
props.put(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE.key(), bootstrapIndexEnable);
return init(getDefaultHadoopConf(), basePath, tableType, props);
}

4
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
View File

@@ -195,9 +195,9 @@ public class TestBootstrap extends HoodieClientTestBase {
private void testBootstrapCommon(boolean partitioned, boolean deltaCommit, EffectiveMode mode) throws Exception {
if (deltaCommit) {
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath);
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath, true);
} else {
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath);
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath, true);
}
int totalRecords = 100;