lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-994] Make TestHBaseQPSResourceAllocator a unit test (#1820)

Browse Source
This commit is contained in:
Raymond Xu
2020-07-11 09:15:05 -07:00
committed by GitHub
parent abfebd30f3
commit 20ac7c3337
1 changed files with 6 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java
View File

@@ -25,45 +25,13 @@ import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.testutils.HoodieClientTestHarness;
import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.testutils.HoodieTestDataGenerator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestHBaseQPSResourceAllocator extends HoodieClientTestHarness { public class TestHBaseQPSResourceAllocator {
private static final String TABLE_NAME = "test_table";
private static final String QPS_TEST_SUFFIX_PATH = "qps_test_suffix";
private HBaseTestingUtility utility;
private Configuration hbaseConfig;
@BeforeEach
public void setUp() throws Exception {
utility = new HBaseTestingUtility();
utility.startMiniCluster();
hbaseConfig = utility.getConnection().getConfiguration();
initSparkContexts("TestQPSResourceAllocator");
initPath();
basePath = tempDir.resolve(QPS_TEST_SUFFIX_PATH).toAbsolutePath().toString();
// Initialize table
initMetaClient();
}
@AfterEach
public void tearDown() throws Exception {
cleanupSparkContexts();
cleanupClients();
if (utility != null) {
utility.shutdownMiniCluster();
}
}
@Test @Test
public void testsDefaultQPSResourceAllocator() { public void testsDefaultQPSResourceAllocator() {
@@ -104,7 +72,7 @@ public class TestHBaseQPSResourceAllocator extends HoodieClientTestHarness {
} }
private HoodieWriteConfig.Builder getConfigBuilder(HoodieHBaseIndexConfig hoodieHBaseIndexConfig) { private HoodieWriteConfig.Builder getConfigBuilder(HoodieHBaseIndexConfig hoodieHBaseIndexConfig) {
return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA) return HoodieWriteConfig.newBuilder().withPath("/foo").withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)
.withParallelism(1, 1) .withParallelism(1, 1)
.withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024 * 1024) .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024 * 1024)
.withInlineCompaction(false).build()) .withInlineCompaction(false).build())
@@ -115,8 +83,10 @@ public class TestHBaseQPSResourceAllocator extends HoodieClientTestHarness {
private HoodieHBaseIndexConfig getConfigWithResourceAllocator(Option<String> resourceAllocatorClass) { private HoodieHBaseIndexConfig getConfigWithResourceAllocator(Option<String> resourceAllocatorClass) {
HoodieHBaseIndexConfig.Builder builder = new HoodieHBaseIndexConfig.Builder() HoodieHBaseIndexConfig.Builder builder = new HoodieHBaseIndexConfig.Builder()
.hbaseZkPort(Integer.parseInt(hbaseConfig.get("hbase.zookeeper.property.clientPort"))) .hbaseZkPort(0)
.hbaseZkQuorum(hbaseConfig.get("hbase.zookeeper.quorum")).hbaseTableName(TABLE_NAME).hbaseIndexGetBatchSize(100); .hbaseZkQuorum("localhost")
.hbaseTableName("foobar")
.hbaseIndexGetBatchSize(100);
if (resourceAllocatorClass.isPresent()) { if (resourceAllocatorClass.isPresent()) {
builder.withQPSResourceAllocatorType(resourceAllocatorClass.get()); builder.withQPSResourceAllocatorType(resourceAllocatorClass.get());
} }