lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2244] Fix database alreadyExists exception while hive sync (#3361)

Browse Source
This commit is contained in:
swuferhong
2021-07-28 19:40:16 +08:00
committed by GitHub
parent 91c2213412
commit eedfadeb46
4 changed files with 56 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
View File

@@ -62,8 +62,10 @@ import java.util.Map;
import static org.apache.hudi.hive.testutils.HiveTestUtil.ddlExecutor;
import static org.apache.hudi.hive.testutils.HiveTestUtil.fileSystem;
import static org.apache.hudi.hive.testutils.HiveTestUtil.hiveSyncConfig;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHiveSyncTool {
@@ -302,6 +304,49 @@ public class TestHiveSyncTool {
"The last commit that was synced should be 100");
}
@ParameterizedTest
@MethodSource({"syncMode"})
public void testSyncDataBase(String syncMode) throws Exception {
hiveSyncConfig.syncMode = syncMode;
HiveTestUtil.hiveSyncConfig.batchSyncNum = 3;
String instantTime = "100";
HiveTestUtil.createCOWTable(instantTime, 5, true);
hiveSyncConfig.databaseName = "database1";
// while autoCreateDatabase is false and database not exists;
hiveSyncConfig.autoCreateDatabase = false;
// Lets do the sync
assertThrows(Exception.class, () -> {
new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable();
});
// while autoCreateDatabase is true and database not exists;
hiveSyncConfig.autoCreateDatabase = true;
HoodieHiveClient hiveClient =
new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem);
assertDoesNotThrow(() -> {
new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable();
});
assertTrue(hiveClient.doesDataBaseExist(hiveSyncConfig.databaseName),
"DataBases " + hiveSyncConfig.databaseName + " should exist after sync completes");
// while autoCreateDatabase is false and database exists;
hiveSyncConfig.autoCreateDatabase = false;
assertDoesNotThrow(() -> {
new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable();
});
assertTrue(hiveClient.doesDataBaseExist(hiveSyncConfig.databaseName),
"DataBases " + hiveSyncConfig.databaseName + " should exist after sync completes");
// while autoCreateDatabase is true and database exists;
hiveSyncConfig.autoCreateDatabase = true;
assertDoesNotThrow(() -> {
new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable();
});
assertTrue(hiveClient.doesDataBaseExist(hiveSyncConfig.databaseName),
"DataBases " + hiveSyncConfig.databaseName + " should exist after sync completes");
}
@ParameterizedTest
@MethodSource({"syncDataSourceTableParams"})
public void testSyncCOWTableWithProperties(boolean useSchemaFromCommitMetadata,
@@ -1054,6 +1099,8 @@ public class TestHiveSyncTool {
hiveSyncConfig.syncMode = syncMode;
HiveTestUtil.hiveSyncConfig.batchSyncNum = 2;
HiveTestUtil.createCOWTable("100", 5, true);
// create database.
ddlExecutor.runSQL("create database " + hiveSyncConfig.databaseName);
HoodieHiveClient hiveClient =
new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem);
String tableName = HiveTestUtil.hiveSyncConfig.tableName;

3
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java
View File

@@ -138,8 +138,7 @@ public class HiveTestUtil {
ddlExecutor.runSQL("drop table if exists " + tableName);
}
createdTablesSet.clear();
ddlExecutor.runSQL("drop database if exists " + hiveSyncConfig.databaseName);
ddlExecutor.runSQL("create database " + hiveSyncConfig.databaseName);
ddlExecutor.runSQL("drop database if exists " + hiveSyncConfig.databaseName + " cascade");
}
public static HiveConf getHiveConf() {