diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index 71f0b994e..4ffb52eaa 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -150,7 +150,9 @@ public class HiveSyncTool extends AbstractSyncTool { // check if the database exists else create it if (cfg.autoCreateDatabase) { try { - hoodieHiveClient.createDatabase(cfg.databaseName); + if (!hoodieHiveClient.doesDataBaseExist(cfg.databaseName)) { + hoodieHiveClient.createDatabase(cfg.databaseName); + } } catch (Exception e) { // this is harmless since table creation will fail anyways, creation of DB is needed for in-memory testing LOG.warn("Unable to create database", e); diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java index 66a41d6f5..36d7c50a5 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -221,14 +220,14 @@ public class HoodieHiveClient extends AbstractSyncHoodieClient { */ public boolean doesDataBaseExist(String databaseName) { try { - Database database = client.getDatabase(databaseName); - if (database != null && databaseName.equals(database.getName())) { - return true; - } + client.getDatabase(databaseName); + return true; + } catch (NoSuchObjectException noSuchObjectException) { + // NoSuchObjectException is thrown when there is no existing database of the name. + return false; } catch (TException e) { throw new HoodieHiveSyncException("Failed to check if database exists " + databaseName, e); } - return false; } public void createDatabase(String databaseName) { diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index af94c16b6..7294f301c 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -62,8 +62,10 @@ import java.util.Map; import static org.apache.hudi.hive.testutils.HiveTestUtil.ddlExecutor; import static org.apache.hudi.hive.testutils.HiveTestUtil.fileSystem; import static org.apache.hudi.hive.testutils.HiveTestUtil.hiveSyncConfig; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHiveSyncTool { @@ -302,6 +304,49 @@ public class TestHiveSyncTool { "The last commit that was synced should be 100"); } + @ParameterizedTest + @MethodSource({"syncMode"}) + public void testSyncDataBase(String syncMode) throws Exception { + hiveSyncConfig.syncMode = syncMode; + HiveTestUtil.hiveSyncConfig.batchSyncNum = 3; + String instantTime = "100"; + HiveTestUtil.createCOWTable(instantTime, 5, true); + hiveSyncConfig.databaseName = "database1"; + + // while autoCreateDatabase is false and database not exists; + hiveSyncConfig.autoCreateDatabase = false; + // Lets do the sync + assertThrows(Exception.class, () -> { + new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable(); + }); + + // while autoCreateDatabase is true and database not exists; + hiveSyncConfig.autoCreateDatabase = true; + HoodieHiveClient hiveClient = + new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem); + assertDoesNotThrow(() -> { + new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable(); + }); + assertTrue(hiveClient.doesDataBaseExist(hiveSyncConfig.databaseName), + "DataBases " + hiveSyncConfig.databaseName + " should exist after sync completes"); + + // while autoCreateDatabase is false and database exists; + hiveSyncConfig.autoCreateDatabase = false; + assertDoesNotThrow(() -> { + new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable(); + }); + assertTrue(hiveClient.doesDataBaseExist(hiveSyncConfig.databaseName), + "DataBases " + hiveSyncConfig.databaseName + " should exist after sync completes"); + + // while autoCreateDatabase is true and database exists; + hiveSyncConfig.autoCreateDatabase = true; + assertDoesNotThrow(() -> { + new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem).syncHoodieTable(); + }); + assertTrue(hiveClient.doesDataBaseExist(hiveSyncConfig.databaseName), + "DataBases " + hiveSyncConfig.databaseName + " should exist after sync completes"); + } + @ParameterizedTest @MethodSource({"syncDataSourceTableParams"}) public void testSyncCOWTableWithProperties(boolean useSchemaFromCommitMetadata, @@ -1054,6 +1099,8 @@ public class TestHiveSyncTool { hiveSyncConfig.syncMode = syncMode; HiveTestUtil.hiveSyncConfig.batchSyncNum = 2; HiveTestUtil.createCOWTable("100", 5, true); + // create database. + ddlExecutor.runSQL("create database " + hiveSyncConfig.databaseName); HoodieHiveClient hiveClient = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); String tableName = HiveTestUtil.hiveSyncConfig.tableName; diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index 4b916bc4b..0686ac0aa 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -138,8 +138,7 @@ public class HiveTestUtil { ddlExecutor.runSQL("drop table if exists " + tableName); } createdTablesSet.clear(); - ddlExecutor.runSQL("drop database if exists " + hiveSyncConfig.databaseName); - ddlExecutor.runSQL("create database " + hiveSyncConfig.databaseName); + ddlExecutor.runSQL("drop database if exists " + hiveSyncConfig.databaseName + " cascade"); } public static HiveConf getHiveConf() {