diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java index 87dd26f0c..69cc25feb 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -1252,7 +1252,7 @@ public class TestCleaner extends HoodieClientTestBase { assertTrue(new File(sourcePath.toString()).exists()); // recreate metaClient with Bootstrap base path - metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString()); + metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString(), true); // generate bootstrap index Map> bootstrapMapping = TestBootstrapIndex.generateBootstrapIndex(metaClient, sourcePath.toString(), diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 6d3a7d001..3d6290a08 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -136,10 +136,10 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { .defaultValue("archived") .withDocumentation("path under the meta folder, to store archived timeline instants at."); - public static final ConfigProperty BOOTSTRAP_INDEX_ENABLE = ConfigProperty + public static final ConfigProperty BOOTSTRAP_INDEX_ENABLE = ConfigProperty .key("hoodie.bootstrap.index.enable") - .noDefaultValue() - .withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined."); + .defaultValue(true) + .withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined, default true."); public static final ConfigProperty BOOTSTRAP_INDEX_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.index.class") @@ -298,8 +298,9 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { } public static String getDefaultBootstrapIndexClass(Properties props) { + HoodieConfig hoodieConfig = new HoodieConfig(props); String defaultClass = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue(); - if ("false".equalsIgnoreCase(props.getProperty(BOOTSTRAP_INDEX_ENABLE.key()))) { + if (!hoodieConfig.getBooleanOrDefault(BOOTSTRAP_INDEX_ENABLE)) { defaultClass = NO_OP_BOOTSTRAP_INDEX_CLASS; } return defaultClass; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 97464b1b2..d5a3988fe 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -627,6 +627,7 @@ public class HoodieTableMetaClient implements Serializable { private String partitionFields; private String bootstrapIndexClass; private String bootstrapBasePath; + private Boolean bootstrapIndexEnable; private Boolean populateMetaFields; private String keyGeneratorClassProp; @@ -702,6 +703,11 @@ public class HoodieTableMetaClient implements Serializable { return this; } + public PropertyBuilder setBootstrapIndexEnable(Boolean bootstrapIndexEnable) { + this.bootstrapIndexEnable = bootstrapIndexEnable; + return this; + } + public PropertyBuilder setPopulateMetaFields(boolean populateMetaFields) { this.populateMetaFields = populateMetaFields; return this; @@ -749,6 +755,11 @@ public class HoodieTableMetaClient implements Serializable { if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_BASE_PATH)) { setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH)); } + + if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE)) { + setBootstrapIndexEnable(hoodieConfig.getBoolean(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE)); + } + if (hoodieConfig.contains(HoodieTableConfig.PRECOMBINE_FIELD)) { setPreCombineField(hoodieConfig.getString(HoodieTableConfig.PRECOMBINE_FIELD)); } @@ -807,6 +818,10 @@ public class HoodieTableMetaClient implements Serializable { tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME, bootstrapIndexClass); } + if (null != bootstrapIndexEnable) { + tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE, Boolean.toString(bootstrapIndexEnable)); + } + if (null != bootstrapBasePath) { tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_BASE_PATH, bootstrapBasePath); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java index dee15e22d..924c6724e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java @@ -108,7 +108,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness { @BeforeEach public void setup() throws IOException { - metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH); + metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, false); basePath = metaClient.getBasePath(); refreshFsView(); } @@ -344,6 +344,11 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness { protected void testViewForFileSlicesWithAsyncCompaction(boolean skipCreatingDataFile, boolean isCompactionInFlight, int expTotalFileSlices, int expTotalDataFiles, boolean includeInvalidAndInflight, boolean testBootstrap) throws Exception { + + if (testBootstrap) { + metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, testBootstrap); + } + String partitionPath = "2016/05/01"; new File(basePath + "/" + partitionPath).mkdirs(); String fileId = UUID.randomUUID().toString(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java index bc1c18a79..259d0c250 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java @@ -63,9 +63,10 @@ public class HoodieTestUtils { return init(getDefaultHadoopConf(), basePath, tableType); } - public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath) throws IOException { + public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath, boolean bootstrapIndexEnable) throws IOException { Properties props = new Properties(); props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath); + props.put(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE.key(), bootstrapIndexEnable); return init(getDefaultHadoopConf(), basePath, tableType, props); } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java index 5cbcf6cf8..59671322b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java @@ -195,9 +195,9 @@ public class TestBootstrap extends HoodieClientTestBase { private void testBootstrapCommon(boolean partitioned, boolean deltaCommit, EffectiveMode mode) throws Exception { if (deltaCommit) { - metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath); + metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath, true); } else { - metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath); + metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath, true); } int totalRecords = 100;