From 441076b2ccf66949310030fc6680bb68c259d8da Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Mon, 7 Jun 2021 15:38:59 -0700 Subject: [PATCH] [HUDI-1950] Move TestHiveMetastoreBasedLockProvider to functional (#3043) HiveTestUtil static setup mini servers caused connection refused issue in Azure CI environment, as TestHiveSyncTool and TestHiveMetastoreBasedLockProvider share the same test facilities. Moving TestHiveMetastoreBasedLockProvider (the easier one) to functional test with a separate and improved mini server setup resolved the issue. Also cleaned up dfs cluster from HiveTestUtil. The next step is to move TestHiveSyncTool to functional as well. --- hudi-sync/hudi-hive-sync/pom.xml | 18 +++ .../apache/hudi/hive/TestHiveSyncTool.java | 5 +- .../HiveSyncFunctionalTestSuite.java | 33 ++++ .../TestHiveMetastoreBasedLockProvider.java | 87 ++++------- .../HiveSyncFunctionalTestHarness.java | 145 ++++++++++++++++++ .../hudi/hive/testutils/HiveTestService.java | 11 +- .../hudi/hive/testutils/HiveTestUtil.java | 32 +--- 7 files changed, 248 insertions(+), 83 deletions(-) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/functional/HiveSyncFunctionalTestSuite.java rename hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/{ => functional}/TestHiveMetastoreBasedLockProvider.java (69%) create mode 100644 hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveSyncFunctionalTestHarness.java diff --git a/hudi-sync/hudi-hive-sync/pom.xml b/hudi-sync/hudi-hive-sync/pom.xml index 44e5f9eee..c44f785e4 100644 --- a/hudi-sync/hudi-hive-sync/pom.xml +++ b/hudi-sync/hudi-hive-sync/pom.xml @@ -178,6 +178,24 @@ test + + org.junit.platform + junit-platform-runner + test + + + + org.junit.platform + junit-platform-suite-api + test + + + + org.junit.platform + junit-platform-commons + test + + org.apache.hadoop diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index 99c599084..118ad121a 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.testutils.NetworkTestUtils; import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.util.Option; import org.apache.hudi.hive.util.ConfigUtils; @@ -245,7 +246,6 @@ public class TestHiveSyncTool { hiveClient = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); List hivePartitions = hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName); List writtenPartitionsSince = hiveClient.getPartitionsWrittenToSince(Option.empty()); - writtenPartitionsSince.add(newPartition.get(0)); List partitionEvents = hiveClient.getPartitionEvents(hivePartitions, writtenPartitionsSince); assertEquals(1, partitionEvents.size(), "There should be only one partition event"); assertEquals(PartitionEventType.UPDATE, partitionEvents.iterator().next().eventType, @@ -687,7 +687,8 @@ public class TestHiveSyncTool { HiveSyncConfig syncToolConfig = HiveSyncConfig.copy(HiveTestUtil.hiveSyncConfig); syncToolConfig.ignoreExceptions = true; - syncToolConfig.jdbcUrl = HiveTestUtil.hiveSyncConfig.jdbcUrl.replace("9999","9031"); + syncToolConfig.jdbcUrl = HiveTestUtil.hiveSyncConfig.jdbcUrl + .replace(String.valueOf(HiveTestUtil.hiveTestService.getHiveServerPort()), String.valueOf(NetworkTestUtils.nextFreePort())); HiveSyncTool tool = new HiveSyncTool(syncToolConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/functional/HiveSyncFunctionalTestSuite.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/functional/HiveSyncFunctionalTestSuite.java new file mode 100644 index 000000000..b194e73b9 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/functional/HiveSyncFunctionalTestSuite.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.hive.functional; + +import org.junit.platform.runner.JUnitPlatform; +import org.junit.platform.suite.api.IncludeTags; +import org.junit.platform.suite.api.SelectPackages; +import org.junit.runner.RunWith; + +@RunWith(JUnitPlatform.class) +@SelectPackages("org.apache.hudi.hive.functional") +@IncludeTags("functional") +public class HiveSyncFunctionalTestSuite { + +} + diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveMetastoreBasedLockProvider.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/functional/TestHiveMetastoreBasedLockProvider.java similarity index 69% rename from hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveMetastoreBasedLockProvider.java rename to hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/functional/TestHiveMetastoreBasedLockProvider.java index 4b1c7f0d7..8f5596a4a 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveMetastoreBasedLockProvider.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/functional/TestHiveMetastoreBasedLockProvider.java @@ -7,32 +7,32 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.hive; +package org.apache.hudi.hive.functional; + +import org.apache.hudi.common.config.LockConfiguration; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.hive.HiveMetastoreBasedLockProvider; +import org.apache.hudi.hive.testutils.HiveSyncFunctionalTestHarness; import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockLevel; import org.apache.hadoop.hive.metastore.api.LockType; -import org.apache.hudi.common.config.LockConfiguration; -import org.apache.hudi.common.config.TypedProperties; -import org.apache.hudi.hive.testutils.HiveTestUtil; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import java.util.concurrent.TimeUnit; import static org.apache.hudi.common.config.LockConfiguration.DEFAULT_LOCK_ACQUIRE_NUM_RETRIES; @@ -54,39 +54,33 @@ import static org.apache.hudi.common.config.LockConfiguration.ZK_SESSION_TIMEOUT * /metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java#L2892 * Unless this is set, we cannot use HiveMetastore server in tests for locking use-cases. */ -public class TestHiveMetastoreBasedLockProvider { +@Tag("functional") +public class TestHiveMetastoreBasedLockProvider extends HiveSyncFunctionalTestHarness { - private static Connection connection; - private static LockComponent lockComponent = new LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, "testdb"); - private static LockConfiguration lockConfiguration; + private static final String TEST_DB_NAME = "testdb"; + private static final String TEST_TABLE_NAME = "testtable"; + private LockComponent lockComponent = new LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, TEST_DB_NAME); + private LockConfiguration lockConfiguration; - @BeforeAll - public static void init() throws Exception { - HiveTestUtil.setUp(); - createHiveConnection(); - connection.createStatement().execute("create database if not exists testdb"); + @BeforeEach + public void init() throws Exception { TypedProperties properties = new TypedProperties(); - properties.setProperty(HIVE_DATABASE_NAME_PROP, "testdb"); - properties.setProperty(HIVE_TABLE_NAME_PROP, "testtable"); + properties.setProperty(HIVE_DATABASE_NAME_PROP, TEST_DB_NAME); + properties.setProperty(HIVE_TABLE_NAME_PROP, TEST_TABLE_NAME); properties.setProperty(LOCK_ACQUIRE_NUM_RETRIES_PROP, DEFAULT_LOCK_ACQUIRE_NUM_RETRIES); properties.setProperty(LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP, DEFAULT_LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS); - properties.setProperty(ZK_CONNECT_URL_PROP, HiveTestUtil.getZkService().connectString()); - properties.setProperty(ZK_PORT_PROP, HiveTestUtil.getHiveConf().get("hive.zookeeper.client.port")); - properties.setProperty(ZK_SESSION_TIMEOUT_MS_PROP, HiveTestUtil.getHiveConf().get("hive.zookeeper.session.timeout")); + properties.setProperty(ZK_CONNECT_URL_PROP, zkService().connectString()); + properties.setProperty(ZK_PORT_PROP, hiveConf().get("hive.zookeeper.client.port")); + properties.setProperty(ZK_SESSION_TIMEOUT_MS_PROP, hiveConf().get("hive.zookeeper.session.timeout")); properties.setProperty(ZK_CONNECTION_TIMEOUT_MS_PROP, String.valueOf(DEFAULT_ZK_CONNECTION_TIMEOUT_MS)); properties.setProperty(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP, String.valueOf(1000)); lockConfiguration = new LockConfiguration(properties); - lockComponent.setTablename("testtable"); - } - - @AfterAll - public static void cleanUpClass() { - HiveTestUtil.shutdown(); + lockComponent.setTablename(TEST_TABLE_NAME); } @Test public void testAcquireLock() throws Exception { - HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, HiveTestUtil.getHiveConf()); + HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, hiveConf()); lockComponent.setOperationType(DataOperationType.NO_TXN); Assertions.assertTrue(lockProvider.acquireLock(lockConfiguration.getConfig() .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP), TimeUnit.MILLISECONDS, lockComponent)); @@ -106,7 +100,7 @@ public class TestHiveMetastoreBasedLockProvider { @Test public void testUnlock() throws Exception { - HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, HiveTestUtil.getHiveConf()); + HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, hiveConf()); lockComponent.setOperationType(DataOperationType.NO_TXN); Assertions.assertTrue(lockProvider.acquireLock(lockConfiguration.getConfig() .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP), TimeUnit.MILLISECONDS, lockComponent)); @@ -119,7 +113,7 @@ public class TestHiveMetastoreBasedLockProvider { @Test public void testReentrantLock() throws Exception { - HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, HiveTestUtil.getHiveConf()); + HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, hiveConf()); lockComponent.setOperationType(DataOperationType.NO_TXN); Assertions.assertTrue(lockProvider.acquireLock(lockConfiguration.getConfig() .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP), TimeUnit.MILLISECONDS, lockComponent)); @@ -135,24 +129,9 @@ public class TestHiveMetastoreBasedLockProvider { @Test public void testUnlockWithoutLock() { - HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, HiveTestUtil.getHiveConf()); + HiveMetastoreBasedLockProvider lockProvider = new HiveMetastoreBasedLockProvider(lockConfiguration, hiveConf()); lockComponent.setOperationType(DataOperationType.NO_TXN); lockProvider.unlock(); } - private static void createHiveConnection() { - if (connection == null) { - try { - Class.forName("org.apache.hive.jdbc.HiveDriver"); - } catch (ClassNotFoundException e) { - throw new RuntimeException(); - } - try { - connection = DriverManager.getConnection("jdbc:hive2://127.0.0.1:9999/"); - } catch (SQLException e) { - throw new HoodieHiveSyncException("Cannot create hive connection ", e); - } - } - } - } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveSyncFunctionalTestHarness.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveSyncFunctionalTestHarness.java new file mode 100644 index 000000000..a145dc024 --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveSyncFunctionalTestHarness.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.hive.testutils; + +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.minicluster.ZookeeperTestService; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveClient; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.time.Instant; +import java.util.Collections; + +public class HiveSyncFunctionalTestHarness { + + private static transient Configuration hadoopConf; + private static transient HiveTestService hiveTestService; + private static transient ZookeeperTestService zookeeperTestService; + + /** + * An indicator of the initialization status. + */ + protected boolean initialized = false; + @TempDir + protected java.nio.file.Path tempDir; + + public String basePath() { + return tempDir.toAbsolutePath().toString(); + } + + public Configuration hadoopConf() { + return hadoopConf; + } + + public FileSystem fs() throws IOException { + return FileSystem.get(hadoopConf); + } + + public HiveTestService hiveService() { + return hiveTestService; + } + + public HiveConf hiveConf() { + return hiveTestService.getHiveServer().getHiveConf(); + } + + public ZookeeperTestService zkService() { + return zookeeperTestService; + } + + public HiveSyncConfig hiveSyncConf() throws IOException { + HiveSyncConfig conf = new HiveSyncConfig(); + conf.jdbcUrl = hiveTestService.getJdbcHive2Url(); + conf.hiveUser = ""; + conf.hivePass = ""; + conf.databaseName = "hivesynctestdb"; + conf.tableName = "hivesynctesttable"; + conf.basePath = Files.createDirectories(tempDir.resolve("hivesynctestcase-" + Instant.now().toEpochMilli())).toUri().toString(); + conf.assumeDatePartitioning = true; + conf.usePreApacheInputFormat = false; + conf.partitionFields = Collections.singletonList("datestr"); + return conf; + } + + public HoodieHiveClient hiveClient(HiveSyncConfig hiveSyncConfig) throws IOException { + HoodieTableMetaClient.withPropertyBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE) + .setTableName(hiveSyncConfig.tableName) + .setPayloadClass(HoodieAvroPayload.class) + .initTable(hadoopConf, hiveSyncConfig.basePath); + return new HoodieHiveClient(hiveSyncConfig, hiveConf(), fs()); + } + + public void dropTables(String database, String... tables) throws IOException { + HiveSyncConfig hiveSyncConfig = hiveSyncConf(); + hiveSyncConfig.databaseName = database; + for (String table : tables) { + hiveSyncConfig.tableName = table; + hiveClient(hiveSyncConfig).updateHiveSQL("drop table if exists " + table); + } + } + + public void dropDatabases(String... databases) throws IOException { + HiveSyncConfig hiveSyncConfig = hiveSyncConf(); + for (String database : databases) { + hiveSyncConfig.databaseName = database; + hiveClient(hiveSyncConfig).updateHiveSQL("drop database if exists " + database); + } + } + + @BeforeEach + public synchronized void runBeforeEach() throws IOException, InterruptedException { + initialized = hiveTestService != null && zookeeperTestService != null; + if (!initialized) { + hadoopConf = new Configuration(); + zookeeperTestService = new ZookeeperTestService(hadoopConf); + zookeeperTestService.start(); + hiveTestService = new HiveTestService(hadoopConf); + hiveTestService.start(); + } + } + + @AfterAll + public static synchronized void cleanUpAfterAll() { + if (hiveTestService != null) { + hiveTestService.stop(); + hiveTestService = null; + } + if (zookeeperTestService != null) { + zookeeperTestService.stop(); + zookeeperTestService = null; + } + if (hadoopConf != null) { + hadoopConf.clear(); + hadoopConf = null; + } + } +} diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java index ca7aa7a02..ac083ab07 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java @@ -79,8 +79,9 @@ public class HiveTestService { private TServer tServer; private HiveServer2 hiveServer; - public HiveTestService(Configuration configuration) throws IOException { + public HiveTestService(Configuration hadoopConf) throws IOException { this.workDir = Files.createTempDirectory(System.currentTimeMillis() + "-").toFile().getAbsolutePath(); + this.hadoopConf = hadoopConf; } public Configuration getHadoopConf() { @@ -152,6 +153,14 @@ public class HiveTestService { return hiveServer; } + public int getHiveServerPort() { + return serverPort; + } + + public String getJdbcHive2Url() { + return String.format("jdbc:hive2://%s:%s/default", bindIP, serverPort); + } + public HiveConf configureHive(Configuration conf, String localHiveLocation) throws IOException { conf.set("hive.metastore.local", "false"); conf.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://" + bindIP + ":" + metastorePort); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index 80a681db4..74f28ccda 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -38,7 +38,6 @@ import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.SchemaTestUtil; -import org.apache.hudi.common.testutils.minicluster.HdfsTestService; import org.apache.hudi.common.testutils.minicluster.ZookeeperTestService; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.hive.HiveSyncConfig; @@ -51,7 +50,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.server.HiveServer2; import org.apache.parquet.avro.AvroSchemaConverter; @@ -67,6 +65,8 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -82,10 +82,9 @@ import static org.junit.jupiter.api.Assertions.fail; @SuppressWarnings("SameParameterValue") public class HiveTestUtil { - private static MiniDFSCluster dfsCluster; private static ZooKeeperServer zkServer; private static HiveServer2 hiveServer; - private static HiveTestService hiveTestService; + public static HiveTestService hiveTestService; private static ZookeeperTestService zkService; private static Configuration configuration; public static HiveSyncConfig hiveSyncConfig; @@ -94,11 +93,7 @@ public class HiveTestUtil { private static Set createdTablesSet = new HashSet<>(); public static void setUp() throws IOException, InterruptedException { - if (dfsCluster == null) { - HdfsTestService service = new HdfsTestService(); - dfsCluster = service.start(true); - configuration = service.getHadoopConf(); - } + configuration = new Configuration(); if (zkServer == null) { zkService = new ZookeeperTestService(configuration); zkServer = zkService.start(); @@ -110,12 +105,12 @@ public class HiveTestUtil { fileSystem = FileSystem.get(configuration); hiveSyncConfig = new HiveSyncConfig(); - hiveSyncConfig.jdbcUrl = "jdbc:hive2://127.0.0.1:9999/"; + hiveSyncConfig.jdbcUrl = hiveTestService.getJdbcHive2Url(); hiveSyncConfig.hiveUser = ""; hiveSyncConfig.hivePass = ""; hiveSyncConfig.databaseName = "testdb"; hiveSyncConfig.tableName = "test1"; - hiveSyncConfig.basePath = "/tmp/hdfs/TestHiveSyncTool/"; + hiveSyncConfig.basePath = Files.createTempDirectory("hivesynctest" + Instant.now().toEpochMilli()).toUri().toString(); hiveSyncConfig.assumeDatePartitioning = true; hiveSyncConfig.usePreApacheInputFormat = false; hiveSyncConfig.partitionFields = Collections.singletonList("datestr"); @@ -146,18 +141,6 @@ public class HiveTestUtil { return hiveServer.getHiveConf(); } - public static HiveServer2 getHiveServer() { - return hiveServer; - } - - public static ZooKeeperServer getZkServer() { - return zkServer; - } - - public static ZookeeperTestService getZkService() { - return zkService; - } - public static void shutdown() { if (hiveServer != null) { hiveServer.stop(); @@ -165,9 +148,6 @@ public class HiveTestUtil { if (hiveTestService != null) { hiveTestService.stop(); } - if (dfsCluster != null) { - dfsCluster.shutdown(); - } if (zkServer != null) { zkServer.shutdown(); }