diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index 66b8ae60c..ab3e8885a 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -26,7 +26,6 @@ import org.apache.hudi.common.model.EmptyHoodieRecordPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; @@ -34,17 +33,10 @@ import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieCompactionConfig; -import org.apache.hudi.config.HoodieHBaseIndexConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIndexException; import org.apache.hudi.index.HoodieIndex.IndexType; -import org.apache.hudi.index.bloom.HoodieBloomIndex; -import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex; -import org.apache.hudi.index.hbase.HBaseIndex; -import org.apache.hudi.index.simple.HoodieSimpleIndex; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; @@ -55,9 +47,7 @@ import org.apache.avro.Schema; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -76,7 +66,6 @@ import scala.Tuple2; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHoodieIndex extends HoodieClientTestHarness { @@ -85,22 +74,19 @@ public class TestHoodieIndex extends HoodieClientTestHarness { private IndexType indexType; private HoodieIndex index; private HoodieWriteConfig config; - private String schemaStr; private Schema schema; private void setUp(IndexType indexType) throws Exception { - setUp(indexType, true); - } - - private void setUp(IndexType indexType, boolean initializeIndex) throws Exception { this.indexType = indexType; initResources(); // We have some records to be tagged (two different partitions) - schemaStr = FileIOUtils.readAsUTFString(getClass().getResourceAsStream("/exampleSchema.txt")); + String schemaStr = FileIOUtils.readAsUTFString(getClass().getResourceAsStream("/exampleSchema.txt")); schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(schemaStr)); - if (initializeIndex) { - instantiateIndex(); - } + config = getConfigBuilder() + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType) + .build()).withAutoCommit(false).build(); + writeClient = getHoodieWriteClient(config); + this.index = writeClient.getIndex(); } @AfterEach @@ -108,75 +94,6 @@ public class TestHoodieIndex extends HoodieClientTestHarness { cleanupResources(); } - @ParameterizedTest - @EnumSource(value = IndexType.class, names = {"BLOOM", "GLOBAL_BLOOM", "SIMPLE", "GLOBAL_SIMPLE", "HBASE"}) - public void testCreateIndex(IndexType indexType) throws Exception { - setUp(indexType, false); - HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder(); - HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder(); - switch (indexType) { - case INMEMORY: - config = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build(); - assertTrue(HoodieIndex.createIndex(config) instanceof InMemoryHashIndex); - break; - case BLOOM: - config = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.BLOOM).build()).build(); - assertTrue(HoodieIndex.createIndex(config) instanceof HoodieBloomIndex); - break; - case GLOBAL_BLOOM: - config = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.GLOBAL_BLOOM).build()).build(); - assertTrue(HoodieIndex.createIndex(config) instanceof HoodieGlobalBloomIndex); - break; - case SIMPLE: - config = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.SIMPLE).build()).build(); - assertTrue(HoodieIndex.createIndex(config) instanceof HoodieSimpleIndex); - break; - case HBASE: - config = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.HBASE) - .withHBaseIndexConfig(new HoodieHBaseIndexConfig.Builder().build()).build()) - .build(); - assertTrue(HoodieIndex.createIndex(config) instanceof HBaseIndex); - break; - default: - // no -op. just for checkstyle errors - } - } - - @Test - public void testCreateDummyIndex() throws Exception { - setUp(IndexType.BLOOM, false); - HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder(); - HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder(); - config = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexClass(DummyHoodieIndex.class.getName()).build()).build(); - assertTrue(HoodieIndex.createIndex(config) instanceof DummyHoodieIndex); - } - - @Test - public void testCreateIndex_withException() throws Exception { - setUp(IndexType.BLOOM, false); - HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder(); - HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder(); - final HoodieWriteConfig config1 = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithConstructor.class.getName()).build()).build(); - final Throwable thrown1 = assertThrows(HoodieException.class, () -> { - HoodieIndex.createIndex(config1); - }, "exception is expected"); - assertTrue(thrown1.getMessage().contains("is not a subclass of HoodieIndex")); - - final HoodieWriteConfig config2 = clientConfigBuilder.withPath(basePath) - .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithoutConstructor.class.getName()).build()).build(); - final Throwable thrown2 = assertThrows(HoodieException.class, () -> { - HoodieIndex.createIndex(config2); - }, "exception is expected"); - assertTrue(thrown2.getMessage().contains("Unable to instantiate class")); - } - @ParameterizedTest @EnumSource(value = IndexType.class, names = {"BLOOM", "GLOBAL_BLOOM", "SIMPLE", "GLOBAL_SIMPLE"}) public void testSimpleTagLocationAndUpdate(IndexType indexType) throws Exception { @@ -537,64 +454,4 @@ public class TestHoodieIndex extends HoodieClientTestHarness { .withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE).build()); } - private void instantiateIndex() { - config = getConfigBuilder() - .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType) - .build()).withAutoCommit(false).build(); - writeClient = getHoodieWriteClient(config); - this.index = writeClient.getIndex(); - } - - public static class DummyHoodieIndex extends HoodieIndex { - - public DummyHoodieIndex(HoodieWriteConfig config) { - super(config); - } - - @Override - public JavaPairRDD>> fetchRecordLocation(JavaRDD hoodieKeys, JavaSparkContext jsc, HoodieTable hoodieTable) { - return null; - } - - @Override - public JavaRDD> tagLocation(JavaRDD> recordRDD, JavaSparkContext jsc, HoodieTable hoodieTable) throws HoodieIndexException { - return null; - } - - @Override - public JavaRDD updateLocation(JavaRDD writeStatusRDD, JavaSparkContext jsc, HoodieTable hoodieTable) throws HoodieIndexException { - return null; - } - - @Override - public boolean rollbackCommit(String instantTime) { - return false; - } - - @Override - public boolean isGlobal() { - return false; - } - - @Override - public boolean canIndexLogFiles() { - return false; - } - - @Override - public boolean isImplicitWithStorage() { - return false; - } - } - - public static class IndexWithConstructor { - - public IndexWithConstructor(HoodieWriteConfig config) { - } - } - - public static class IndexWithoutConstructor { - - } - } diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndexConfigs.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndexConfigs.java new file mode 100644 index 000000000..a68a8720c --- /dev/null +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndexConfigs.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.index; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieHBaseIndexConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIndexException; +import org.apache.hudi.index.HoodieIndex.IndexType; +import org.apache.hudi.index.bloom.HoodieBloomIndex; +import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex; +import org.apache.hudi.index.hbase.HBaseIndex; +import org.apache.hudi.index.simple.HoodieSimpleIndex; +import org.apache.hudi.table.HoodieTable; + +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestHoodieIndexConfigs { + + private String basePath; + + @BeforeEach + public void setUp(@TempDir Path tempDir) { + basePath = tempDir.toString(); + } + + @ParameterizedTest + @EnumSource(value = IndexType.class, names = {"BLOOM", "GLOBAL_BLOOM", "SIMPLE", "GLOBAL_SIMPLE", "HBASE"}) + public void testCreateIndex(IndexType indexType) throws Exception { + HoodieWriteConfig config; + HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder(); + HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder(); + switch (indexType) { + case INMEMORY: + config = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build(); + assertTrue(HoodieIndex.createIndex(config) instanceof InMemoryHashIndex); + break; + case BLOOM: + config = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.BLOOM).build()).build(); + assertTrue(HoodieIndex.createIndex(config) instanceof HoodieBloomIndex); + break; + case GLOBAL_BLOOM: + config = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.GLOBAL_BLOOM).build()).build(); + assertTrue(HoodieIndex.createIndex(config) instanceof HoodieGlobalBloomIndex); + break; + case SIMPLE: + config = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.SIMPLE).build()).build(); + assertTrue(HoodieIndex.createIndex(config) instanceof HoodieSimpleIndex); + break; + case HBASE: + config = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.HBASE) + .withHBaseIndexConfig(new HoodieHBaseIndexConfig.Builder().build()).build()) + .build(); + assertTrue(HoodieIndex.createIndex(config) instanceof HBaseIndex); + break; + default: + // no -op. just for checkstyle errors + } + } + + @Test + public void testCreateDummyIndex() { + HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder(); + HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder(); + HoodieWriteConfig config = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexClass(DummyHoodieIndex.class.getName()).build()).build(); + assertTrue(HoodieIndex.createIndex(config) instanceof DummyHoodieIndex); + } + + @Test + public void testCreateIndexWithException() { + HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder(); + HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder(); + final HoodieWriteConfig config1 = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithConstructor.class.getName()).build()).build(); + final Throwable thrown1 = assertThrows(HoodieException.class, () -> { + HoodieIndex.createIndex(config1); + }, "exception is expected"); + assertTrue(thrown1.getMessage().contains("is not a subclass of HoodieIndex")); + + final HoodieWriteConfig config2 = clientConfigBuilder.withPath(basePath) + .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithoutConstructor.class.getName()).build()).build(); + final Throwable thrown2 = assertThrows(HoodieException.class, () -> { + HoodieIndex.createIndex(config2); + }, "exception is expected"); + assertTrue(thrown2.getMessage().contains("Unable to instantiate class")); + } + + public static class DummyHoodieIndex> extends HoodieIndex { + + public DummyHoodieIndex(HoodieWriteConfig config) { + super(config); + } + + @Override + public JavaPairRDD>> fetchRecordLocation(JavaRDD hoodieKeys, JavaSparkContext jsc, HoodieTable hoodieTable) { + return null; + } + + @Override + public JavaRDD> tagLocation(JavaRDD> recordRDD, JavaSparkContext jsc, HoodieTable hoodieTable) throws HoodieIndexException { + return null; + } + + @Override + public JavaRDD updateLocation(JavaRDD writeStatusRDD, JavaSparkContext jsc, HoodieTable hoodieTable) throws HoodieIndexException { + return null; + } + + @Override + public boolean rollbackCommit(String instantTime) { + return false; + } + + @Override + public boolean isGlobal() { + return false; + } + + @Override + public boolean canIndexLogFiles() { + return false; + } + + @Override + public boolean isImplicitWithStorage() { + return false; + } + } + + public static class IndexWithConstructor { + + public IndexWithConstructor(HoodieWriteConfig config) { + } + } + + public static class IndexWithoutConstructor { + + } +}