lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Fix CI issue with TestHiveSyncTool (#6110)

Browse Source
This commit is contained in:
Shiyan Xu
2022-07-22 10:30:00 -05:00
committed by GitHub
parent 41653fc708
commit d5c904e10e
10 changed files with 101 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/HoodieDeltaStreamerTestBase.java
View File

@@ -48,6 +48,7 @@ import java.util.Map;
import java.util.Random;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_URL;
import static org.apache.hudi.hive.testutils.HiveTestService.HS2_JDBC_URL;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_ASSUME_DATE_PARTITION;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS;
@@ -186,7 +187,7 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
props.setProperty("hoodie.deltastreamer.schemaprovider.target.schema.file", dfsBasePath + "/target.avsc");
// Hive Configs
props.setProperty(HIVE_URL.key(), "jdbc:hive2://127.0.0.1:9999/");
props.setProperty(HIVE_URL.key(), HS2_JDBC_URL);
props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb1");
props.setProperty(META_SYNC_TABLE_NAME.key(), "hive_trips");
props.setProperty(META_SYNC_PARTITION_FIELDS.key(), "datestr");
@@ -246,7 +247,7 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
protected static void populateCommonHiveProps(TypedProperties props) {
// Hive Configs
props.setProperty(HIVE_URL.key(), "jdbc:hive2://127.0.0.1:9999/");
props.setProperty(HIVE_URL.key(), HS2_JDBC_URL);
props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb2");
props.setProperty(META_SYNC_ASSUME_DATE_PARTITION.key(), "false");
props.setProperty(META_SYNC_PARTITION_FIELDS.key(), "datestr");

2
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
View File

@@ -1359,7 +1359,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
// Test Hive integration
HiveSyncConfig hiveSyncConfig = getHiveSyncConfig(tableBasePath, "hive_trips");
hiveSyncConfig.setValue(META_SYNC_PARTITION_FIELDS, "year,month,day");
hiveSyncConfig.setHadoopConf(hiveServer.getHiveConf());
hiveSyncConfig.setHadoopConf(hiveTestService.getHiveConf());
HoodieHiveSyncClient hiveClient = new HoodieHiveSyncClient(hiveSyncConfig);
final String tableName = hiveSyncConfig.getString(META_SYNC_TABLE_NAME);
assertTrue(hiveClient.tableExists(tableName), "Table " + tableName + " should exist");

7
hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
View File

@@ -56,7 +56,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.server.HiveServer2;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -197,7 +196,7 @@ public class UtilitiesTestBase {
*/
protected static HiveSyncConfig getHiveSyncConfig(String basePath, String tableName) {
Properties props = new Properties();
props.setProperty(HIVE_URL.key(),"jdbc:hive2://127.0.0.1:9999/");
props.setProperty(HIVE_URL.key(), hiveTestService.getJdbcHive2Url());
props.setProperty(HIVE_USER.key(), "");
props.setProperty(HIVE_PASS.key(), "");
props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb1");
@@ -215,11 +214,9 @@ public class UtilitiesTestBase {
* @throws IOException
*/
private static void clearHiveDb() throws Exception {
HiveConf hiveConf = new HiveConf();
// Create Dummy hive sync config
HiveSyncConfig hiveSyncConfig = getHiveSyncConfig("/dummy", "dummy");
hiveConf.addResource(hiveServer.getHiveConf());
hiveSyncConfig.setHadoopConf(hiveConf);
hiveSyncConfig.setHadoopConf(hiveTestService.getHiveConf());
HoodieTableMetaClient.withPropertyBuilder()
.setTableType(HoodieTableType.COPY_ON_WRITE)
.setTableName(hiveSyncConfig.getString(META_SYNC_TABLE_NAME))