1
0

[HUDI-2410] Fix getDefaultBootstrapIndexClass logical error (#3633)

This commit is contained in:
liujinhui
2021-09-13 16:10:17 +08:00
committed by GitHub
parent c79017cb74
commit 9f3c4a2a7f
6 changed files with 31 additions and 9 deletions

View File

@@ -1252,7 +1252,7 @@ public class TestCleaner extends HoodieClientTestBase {
assertTrue(new File(sourcePath.toString()).exists()); assertTrue(new File(sourcePath.toString()).exists());
// recreate metaClient with Bootstrap base path // recreate metaClient with Bootstrap base path
metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString()); metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString(), true);
// generate bootstrap index // generate bootstrap index
Map<String, List<BootstrapFileMapping>> bootstrapMapping = TestBootstrapIndex.generateBootstrapIndex(metaClient, sourcePath.toString(), Map<String, List<BootstrapFileMapping>> bootstrapMapping = TestBootstrapIndex.generateBootstrapIndex(metaClient, sourcePath.toString(),

View File

@@ -136,10 +136,10 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable {
.defaultValue("archived") .defaultValue("archived")
.withDocumentation("path under the meta folder, to store archived timeline instants at."); .withDocumentation("path under the meta folder, to store archived timeline instants at.");
public static final ConfigProperty<String> BOOTSTRAP_INDEX_ENABLE = ConfigProperty public static final ConfigProperty<Boolean> BOOTSTRAP_INDEX_ENABLE = ConfigProperty
.key("hoodie.bootstrap.index.enable") .key("hoodie.bootstrap.index.enable")
.noDefaultValue() .defaultValue(true)
.withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined."); .withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined, default true.");
public static final ConfigProperty<String> BOOTSTRAP_INDEX_CLASS_NAME = ConfigProperty public static final ConfigProperty<String> BOOTSTRAP_INDEX_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.index.class") .key("hoodie.bootstrap.index.class")
@@ -298,8 +298,9 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable {
} }
public static String getDefaultBootstrapIndexClass(Properties props) { public static String getDefaultBootstrapIndexClass(Properties props) {
HoodieConfig hoodieConfig = new HoodieConfig(props);
String defaultClass = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue(); String defaultClass = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue();
if ("false".equalsIgnoreCase(props.getProperty(BOOTSTRAP_INDEX_ENABLE.key()))) { if (!hoodieConfig.getBooleanOrDefault(BOOTSTRAP_INDEX_ENABLE)) {
defaultClass = NO_OP_BOOTSTRAP_INDEX_CLASS; defaultClass = NO_OP_BOOTSTRAP_INDEX_CLASS;
} }
return defaultClass; return defaultClass;

View File

@@ -627,6 +627,7 @@ public class HoodieTableMetaClient implements Serializable {
private String partitionFields; private String partitionFields;
private String bootstrapIndexClass; private String bootstrapIndexClass;
private String bootstrapBasePath; private String bootstrapBasePath;
private Boolean bootstrapIndexEnable;
private Boolean populateMetaFields; private Boolean populateMetaFields;
private String keyGeneratorClassProp; private String keyGeneratorClassProp;
@@ -702,6 +703,11 @@ public class HoodieTableMetaClient implements Serializable {
return this; return this;
} }
public PropertyBuilder setBootstrapIndexEnable(Boolean bootstrapIndexEnable) {
this.bootstrapIndexEnable = bootstrapIndexEnable;
return this;
}
public PropertyBuilder setPopulateMetaFields(boolean populateMetaFields) { public PropertyBuilder setPopulateMetaFields(boolean populateMetaFields) {
this.populateMetaFields = populateMetaFields; this.populateMetaFields = populateMetaFields;
return this; return this;
@@ -749,6 +755,11 @@ public class HoodieTableMetaClient implements Serializable {
if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_BASE_PATH)) { if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_BASE_PATH)) {
setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH)); setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH));
} }
if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE)) {
setBootstrapIndexEnable(hoodieConfig.getBoolean(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE));
}
if (hoodieConfig.contains(HoodieTableConfig.PRECOMBINE_FIELD)) { if (hoodieConfig.contains(HoodieTableConfig.PRECOMBINE_FIELD)) {
setPreCombineField(hoodieConfig.getString(HoodieTableConfig.PRECOMBINE_FIELD)); setPreCombineField(hoodieConfig.getString(HoodieTableConfig.PRECOMBINE_FIELD));
} }
@@ -807,6 +818,10 @@ public class HoodieTableMetaClient implements Serializable {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME, bootstrapIndexClass); tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME, bootstrapIndexClass);
} }
if (null != bootstrapIndexEnable) {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE, Boolean.toString(bootstrapIndexEnable));
}
if (null != bootstrapBasePath) { if (null != bootstrapBasePath) {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_BASE_PATH, bootstrapBasePath); tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_BASE_PATH, bootstrapBasePath);
} }

View File

@@ -108,7 +108,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
@BeforeEach @BeforeEach
public void setup() throws IOException { public void setup() throws IOException {
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH); metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, false);
basePath = metaClient.getBasePath(); basePath = metaClient.getBasePath();
refreshFsView(); refreshFsView();
} }
@@ -344,6 +344,11 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
protected void testViewForFileSlicesWithAsyncCompaction(boolean skipCreatingDataFile, boolean isCompactionInFlight, protected void testViewForFileSlicesWithAsyncCompaction(boolean skipCreatingDataFile, boolean isCompactionInFlight,
int expTotalFileSlices, int expTotalDataFiles, boolean includeInvalidAndInflight, boolean testBootstrap) int expTotalFileSlices, int expTotalDataFiles, boolean includeInvalidAndInflight, boolean testBootstrap)
throws Exception { throws Exception {
if (testBootstrap) {
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, testBootstrap);
}
String partitionPath = "2016/05/01"; String partitionPath = "2016/05/01";
new File(basePath + "/" + partitionPath).mkdirs(); new File(basePath + "/" + partitionPath).mkdirs();
String fileId = UUID.randomUUID().toString(); String fileId = UUID.randomUUID().toString();

View File

@@ -63,9 +63,10 @@ public class HoodieTestUtils {
return init(getDefaultHadoopConf(), basePath, tableType); return init(getDefaultHadoopConf(), basePath, tableType);
} }
public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath) throws IOException { public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath, boolean bootstrapIndexEnable) throws IOException {
Properties props = new Properties(); Properties props = new Properties();
props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath); props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath);
props.put(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE.key(), bootstrapIndexEnable);
return init(getDefaultHadoopConf(), basePath, tableType, props); return init(getDefaultHadoopConf(), basePath, tableType, props);
} }

View File

@@ -195,9 +195,9 @@ public class TestBootstrap extends HoodieClientTestBase {
private void testBootstrapCommon(boolean partitioned, boolean deltaCommit, EffectiveMode mode) throws Exception { private void testBootstrapCommon(boolean partitioned, boolean deltaCommit, EffectiveMode mode) throws Exception {
if (deltaCommit) { if (deltaCommit) {
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath); metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath, true);
} else { } else {
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath); metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath, true);
} }
int totalRecords = 100; int totalRecords = 100;