lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1446] Support skip bootstrapIndex's init in abstract fs view init (#2520)

Browse Source 
Co-authored-by: zhongliang <zhongliang@kuaishou.com>
Co-authored-by: Sivabalan Narayanan <sivabala@uber.com>
This commit is contained in:
xoln ann
2021-05-14 12:29:26 +08:00
committed by GitHub
parent ad77cf42ba
commit 12443e4187
6 changed files with 93 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java
View File

@@ -24,6 +24,7 @@ import org.apache.hudi.avro.model.HoodiePath;
import org.apache.hudi.common.bootstrap.index.BootstrapIndex;
import org.apache.hudi.common.bootstrap.index.BootstrapIndex.IndexWriter;
import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex;
import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex;
import org.apache.hudi.common.model.BootstrapFileMapping;
import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.common.table.HoodieTableConfig;
@@ -31,6 +32,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import java.io.IOException;
@@ -41,6 +43,7 @@ import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
@@ -86,6 +89,21 @@ public class TestBootstrapIndex extends HoodieCommonTestHarness {
testBootstrapIndexOneRound(5);
}
@Test
public void testNoOpBootstrapIndex() throws IOException {
Map<String, String> props = metaClient.getTableConfig().getProps();
props.put(HoodieTableConfig.HOODIE_BOOTSTRAP_INDEX_ENABLE, "false");
Properties properties = new Properties();
for (Map.Entry<String, String> prop : props.entrySet()) {
properties.setProperty(prop.getKey(), prop.getValue());
}
HoodieTableConfig.createHoodieProperties(metaClient.getFs(), new Path(metaClient.getMetaPath()), properties);
metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).build();
BootstrapIndex bootstrapIndex = BootstrapIndex.getBootstrapIndex(metaClient);
assert (bootstrapIndex instanceof NoOpBootstrapIndex);
}
@Test
public void testBootstrapIndexConcurrent() throws Exception {
Map<String, List<BootstrapFileMapping>> bootstrapMapping = generateBootstrapIndex(metaClient, BOOTSTRAP_BASE_PATH, PARTITIONS, 100);