diff --git a/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java b/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java index a0b4be835..47a0214c5 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java @@ -119,6 +119,6 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl } protected HoodieTableMetaClient createMetaClient(boolean loadActiveTimelineOnLoad) { - return ClientUtils.createMetaClient(jsc, config, loadActiveTimelineOnLoad); + return ClientUtils.createMetaClient(jsc.hadoopConfiguration(), config, loadActiveTimelineOnLoad); } } diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java index 4b97c4ae1..1eee57add 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java @@ -97,7 +97,7 @@ public class HoodieReadClient implements Serializ // Create a Hoodie table which encapsulated the commits and files visible HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true); this.hoodieTable = HoodieTable.create(metaClient, clientConfig, jsc); - this.index = HoodieIndex.createIndex(clientConfig, jsc); + this.index = HoodieIndex.createIndex(clientConfig); this.sqlContextOpt = Option.empty(); } diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java index 1d09060bc..984769e5f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java @@ -98,7 +98,7 @@ public class HoodieWriteClient extends AbstractHo * @param rollbackPending whether need to cleanup pending commits */ public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending) { - this(jsc, clientConfig, rollbackPending, HoodieIndex.createIndex(clientConfig, jsc)); + this(jsc, clientConfig, rollbackPending, HoodieIndex.createIndex(clientConfig)); } HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending, HoodieIndex index) { diff --git a/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java index 47003ba3d..0cfe0c1bb 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java @@ -18,25 +18,24 @@ package org.apache.hudi.client.utils; +import org.apache.hadoop.conf.Configuration; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.spark.api.java.JavaSparkContext; - public class ClientUtils { /** * Create Consistency Aware MetaClient. * - * @param jsc JavaSparkContext + * @param hadoopConf Configuration * @param config HoodieWriteConfig * @param loadActiveTimelineOnLoad early loading of timeline */ - public static HoodieTableMetaClient createMetaClient(JavaSparkContext jsc, HoodieWriteConfig config, + public static HoodieTableMetaClient createMetaClient(Configuration hadoopConf, HoodieWriteConfig config, boolean loadActiveTimelineOnLoad) { - return new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), loadActiveTimelineOnLoad, + return new HoodieTableMetaClient(hadoopConf, config.getBasePath(), loadActiveTimelineOnLoad, config.getConsistencyGuardConfig(), Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion()))); } diff --git a/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java index 645184ea0..93fcc8930 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java @@ -51,8 +51,8 @@ public abstract class HoodieIndex implements Seri this.config = config; } - public static HoodieIndex createIndex(HoodieWriteConfig config, - JavaSparkContext jsc) throws HoodieIndexException { + public static HoodieIndex createIndex( + HoodieWriteConfig config) throws HoodieIndexException { // first use index class config to create index. if (!StringUtils.isNullOrEmpty(config.getIndexClass())) { Object instance = ReflectionUtils.loadClass(config.getIndexClass(), config); diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java index 62509e429..ef156ad82 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -95,7 +95,7 @@ public abstract class HoodieTable implements Seri this.viewManager = FileSystemViewManager.createViewManager(new SerializableConfiguration(jsc.hadoopConfiguration()), config.getViewStorageConfig()); this.metaClient = metaClient; - this.index = HoodieIndex.createIndex(config, jsc); + this.index = HoodieIndex.createIndex(config); } private synchronized FileSystemViewManager getViewManager() { diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java index c18993191..899c8686c 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java @@ -89,7 +89,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { } protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit) { - return getHoodieWriteClient(cfg, rollbackInflightCommit, HoodieIndex.createIndex(cfg, jsc)); + return getHoodieWriteClient(cfg, rollbackInflightCommit, HoodieIndex.createIndex(cfg)); } protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit, @@ -247,7 +247,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { private Function2, String, Integer> wrapRecordsGenFunctionForPreppedCalls( final HoodieWriteConfig writeConfig, final Function2, String, Integer> recordGenFunction) { return (commit, numRecords) -> { - final HoodieIndex index = HoodieIndex.createIndex(writeConfig, jsc); + final HoodieIndex index = HoodieIndex.createIndex(writeConfig); List records = recordGenFunction.apply(commit, numRecords); final HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true); HoodieTable table = HoodieTable.create(metaClient, writeConfig, jsc); @@ -268,7 +268,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { private Function> wrapDeleteKeysGenFunctionForPreppedCalls( final HoodieWriteConfig writeConfig, final Function> keyGenFunction) { return (numRecords) -> { - final HoodieIndex index = HoodieIndex.createIndex(writeConfig, jsc); + final HoodieIndex index = HoodieIndex.createIndex(writeConfig); List records = keyGenFunction.apply(numRecords); final HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true); HoodieTable table = HoodieTable.create(metaClient, writeConfig, jsc); diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index 17b58d626..9e93da305 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -73,20 +73,20 @@ public class TestHoodieIndex extends HoodieClientTestHarness { .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.HBASE) .withHBaseIndexConfig(new HoodieHBaseIndexConfig.Builder().build()).build()) .build(); - assertTrue(HoodieIndex.createIndex(config, jsc) instanceof HBaseIndex); + assertTrue(HoodieIndex.createIndex(config) instanceof HBaseIndex); config = clientConfigBuilder.withPath(basePath) .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build(); - assertTrue(HoodieIndex.createIndex(config, jsc) instanceof InMemoryHashIndex); + assertTrue(HoodieIndex.createIndex(config) instanceof InMemoryHashIndex); config = clientConfigBuilder.withPath(basePath) .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.BLOOM).build()).build(); - assertTrue(HoodieIndex.createIndex(config, jsc) instanceof HoodieBloomIndex); + assertTrue(HoodieIndex.createIndex(config) instanceof HoodieBloomIndex); config = clientConfigBuilder.withPath(basePath) .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.GLOBAL_BLOOM).build()).build(); - assertTrue(HoodieIndex.createIndex(config, jsc) instanceof HoodieGlobalBloomIndex); + assertTrue(HoodieIndex.createIndex(config) instanceof HoodieGlobalBloomIndex); config = clientConfigBuilder.withPath(basePath) .withIndexConfig(indexConfigBuilder.withIndexClass(DummyHoodieIndex.class.getName()).build()).build(); - assertTrue(HoodieIndex.createIndex(config, jsc) instanceof DummyHoodieIndex); + assertTrue(HoodieIndex.createIndex(config) instanceof DummyHoodieIndex); } @Test @@ -94,14 +94,14 @@ public class TestHoodieIndex extends HoodieClientTestHarness { final HoodieWriteConfig config1 = clientConfigBuilder.withPath(basePath) .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithConstructor.class.getName()).build()).build(); final Throwable thrown1 = assertThrows(HoodieException.class, () -> { - HoodieIndex.createIndex(config1, jsc); + HoodieIndex.createIndex(config1); }, "exception is expected"); assertTrue(thrown1.getMessage().contains("is not a subclass of HoodieIndex")); final HoodieWriteConfig config2 = clientConfigBuilder.withPath(basePath) .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithoutConstructor.class.getName()).build()).build(); final Throwable thrown2 = assertThrows(HoodieException.class, () -> { - HoodieIndex.createIndex(config2, jsc); + HoodieIndex.createIndex(config2); }, "exception is expected"); assertTrue(thrown2.getMessage().contains("Unable to instantiate class")); } diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java index 296530bd1..5c62423c8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -138,7 +138,7 @@ public class TestCleaner extends TestHoodieClientBase { "The clean instant should be the same as the commit instant"); } - HoodieIndex index = HoodieIndex.createIndex(cfg, jsc); + HoodieIndex index = HoodieIndex.createIndex(cfg); List taggedRecords = index.tagLocation(jsc.parallelize(records, 1), jsc, table).collect(); checkTaggedRecords(taggedRecords, newCommitTime); }