lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-870] Remove spark context in ClientUtils and HoodieIndex (#1609)

Browse Source
This commit is contained in:
Shen Hong
2020-05-11 19:05:36 +08:00
committed by GitHub
parent 8d0e23173b
commit 6dac10115c
9 changed files with 21 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java
View File

@@ -89,7 +89,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
}
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit) {
return getHoodieWriteClient(cfg, rollbackInflightCommit, HoodieIndex.createIndex(cfg, jsc));
return getHoodieWriteClient(cfg, rollbackInflightCommit, HoodieIndex.createIndex(cfg));
}
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit,
@@ -247,7 +247,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
private Function2<List<HoodieRecord>, String, Integer> wrapRecordsGenFunctionForPreppedCalls(
final HoodieWriteConfig writeConfig, final Function2<List<HoodieRecord>, String, Integer> recordGenFunction) {
return (commit, numRecords) -> {
final HoodieIndex index = HoodieIndex.createIndex(writeConfig, jsc);
final HoodieIndex index = HoodieIndex.createIndex(writeConfig);
List<HoodieRecord> records = recordGenFunction.apply(commit, numRecords);
final HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true);
HoodieTable table = HoodieTable.create(metaClient, writeConfig, jsc);
@@ -268,7 +268,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
private Function<Integer, List<HoodieKey>> wrapDeleteKeysGenFunctionForPreppedCalls(
final HoodieWriteConfig writeConfig, final Function<Integer, List<HoodieKey>> keyGenFunction) {
return (numRecords) -> {
final HoodieIndex index = HoodieIndex.createIndex(writeConfig, jsc);
final HoodieIndex index = HoodieIndex.createIndex(writeConfig);
List<HoodieKey> records = keyGenFunction.apply(numRecords);
final HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true);
HoodieTable table = HoodieTable.create(metaClient, writeConfig, jsc);

14
hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java
View File

@@ -73,20 +73,20 @@ public class TestHoodieIndex extends HoodieClientTestHarness {
.withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.HBASE)
.withHBaseIndexConfig(new HoodieHBaseIndexConfig.Builder().build()).build())
.build();
assertTrue(HoodieIndex.createIndex(config, jsc) instanceof HBaseIndex);
assertTrue(HoodieIndex.createIndex(config) instanceof HBaseIndex);
config = clientConfigBuilder.withPath(basePath)
.withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
assertTrue(HoodieIndex.createIndex(config, jsc) instanceof InMemoryHashIndex);
assertTrue(HoodieIndex.createIndex(config) instanceof InMemoryHashIndex);
config = clientConfigBuilder.withPath(basePath)
.withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
assertTrue(HoodieIndex.createIndex(config, jsc) instanceof HoodieBloomIndex);
assertTrue(HoodieIndex.createIndex(config) instanceof HoodieBloomIndex);
config = clientConfigBuilder.withPath(basePath)
.withIndexConfig(indexConfigBuilder.withIndexType(IndexType.GLOBAL_BLOOM).build()).build();
assertTrue(HoodieIndex.createIndex(config, jsc) instanceof HoodieGlobalBloomIndex);
assertTrue(HoodieIndex.createIndex(config) instanceof HoodieGlobalBloomIndex);
config = clientConfigBuilder.withPath(basePath)
.withIndexConfig(indexConfigBuilder.withIndexClass(DummyHoodieIndex.class.getName()).build()).build();
assertTrue(HoodieIndex.createIndex(config, jsc) instanceof DummyHoodieIndex);
assertTrue(HoodieIndex.createIndex(config) instanceof DummyHoodieIndex);
}
@Test
@@ -94,14 +94,14 @@ public class TestHoodieIndex extends HoodieClientTestHarness {
final HoodieWriteConfig config1 = clientConfigBuilder.withPath(basePath)
.withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithConstructor.class.getName()).build()).build();
final Throwable thrown1 = assertThrows(HoodieException.class, () -> {
HoodieIndex.createIndex(config1, jsc);
HoodieIndex.createIndex(config1);
}, "exception is expected");
assertTrue(thrown1.getMessage().contains("is not a subclass of HoodieIndex"));
final HoodieWriteConfig config2 = clientConfigBuilder.withPath(basePath)
.withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithoutConstructor.class.getName()).build()).build();
final Throwable thrown2 = assertThrows(HoodieException.class, () -> {
HoodieIndex.createIndex(config2, jsc);
HoodieIndex.createIndex(config2);
}, "exception is expected");
assertTrue(thrown2.getMessage().contains("Unable to instantiate class"));
}

2
hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
View File

@@ -138,7 +138,7 @@ public class TestCleaner extends TestHoodieClientBase {
"The clean instant should be the same as the commit instant");
}
HoodieIndex index = HoodieIndex.createIndex(cfg, jsc);
HoodieIndex index = HoodieIndex.createIndex(cfg);
List<HoodieRecord> taggedRecords = index.tagLocation(jsc.parallelize(records, 1), jsc, table).collect();
checkTaggedRecords(taggedRecords, newCommitTime);
}