lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3091] Making SIMPLE index as the default index type (#4659)

Browse Source 
* [HUDI-3091] Making SIMPLE index as the default index type

* Fixing tests

* Traiging timeouts

* disable SIMPLE index for bootstrap tests

* removing test run start and end log statements

* Fixing simple index parallellism for some tests

* Disabling failing test for now

* reverting previous disable

* Reverting all changes

* fixing azure pipeline script
This commit is contained in:
Sivabalan Narayanan
2022-02-08 04:32:18 -05:00
committed by GitHub
parent ab73047958
commit 6a32cfe020
3 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
View File

@@ -152,7 +152,7 @@ public class HoodieIndexConfig extends HoodieConfig {
public static final ConfigProperty<String> SIMPLE_INDEX_PARALLELISM = ConfigProperty
.key("hoodie.simple.index.parallelism")
.defaultValue("50")
.defaultValue("100")
.withDocumentation("Only applies if index type is SIMPLE. "
+ "This is the amount of parallelism for index lookup, which involves a Spark Shuffle");
@@ -568,7 +568,7 @@ public class HoodieIndexConfig extends HoodieConfig {
private String getDefaultIndexType(EngineType engineType) {
switch (engineType) {
case SPARK:
return HoodieIndex.IndexType.BLOOM.name();
return HoodieIndex.IndexType.SIMPLE.name();
case FLINK:
case JAVA:
return HoodieIndex.IndexType.INMEMORY.name();

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieSimpleIndex.java
View File

@@ -148,7 +148,7 @@ public class HoodieSimpleIndex
protected HoodiePairData<HoodieKey, HoodieRecordLocation> fetchRecordLocations(
HoodieEngineContext context, HoodieTable hoodieTable, int parallelism,
List<Pair<String, HoodieBaseFile>> baseFiles) {
int fetchParallelism = Math.max(1, Math.max(baseFiles.size(), parallelism));
int fetchParallelism = Math.max(1, Math.min(baseFiles.size(), parallelism));
return context.parallelize(baseFiles, fetchParallelism)
.flatMap(partitionPathBaseFile -> new HoodieKeyLocationFetchHandle(config, hoodieTable, partitionPathBaseFile, keyGeneratorOpt)

2
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
View File

@@ -82,7 +82,7 @@ public class TestHoodieWriteConfig {
public void testDefaultIndexAccordingToEngineType() {
testEngineSpecificConfig(HoodieWriteConfig::getIndexType,
constructConfigMap(
EngineType.SPARK, HoodieIndex.IndexType.BLOOM,
EngineType.SPARK, HoodieIndex.IndexType.SIMPLE,
EngineType.FLINK, HoodieIndex.IndexType.INMEMORY,
EngineType.JAVA, HoodieIndex.IndexType.INMEMORY));
}