[HUDI-2362] Add external config file support (#3416)

Co-authored-by: Wenning Ding <wenningd@amazon.com>

hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java

@@ -25,10 +25,14 @@ import org.apache.hudi.common.testutils.minicluster.HdfsTestService;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.Rule;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
 import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
 
@@ -47,6 +51,10 @@ public class TestDFSPropertiesConfiguration {
   private static MiniDFSCluster dfsCluster;
   private static DistributedFileSystem dfs;
 
+  @Rule
+  public static final EnvironmentVariables ENVIRONMENT_VARIABLES
+      = new EnvironmentVariables();
+
   @BeforeAll
   public static void initClass() throws Exception {
     hdfsTestService = new HdfsTestService();
@@ -73,12 +81,17 @@ public class TestDFSPropertiesConfiguration {
   }
 
   @AfterAll
-  public static void cleanupClass() throws Exception {
+  public static void cleanupClass() {
     if (hdfsTestService != null) {
       hdfsTestService.stop();
     }
   }
 
+  @AfterEach
+  public void cleanupGlobalConfig() {
+    DFSPropertiesConfiguration.clearGlobalProps();
+  }
+
   private static void writePropertiesFile(Path path, String[] lines) throws IOException {
     PrintStream out = new PrintStream(dfs.create(path, true));
     for (String line : lines) {
@@ -91,7 +104,7 @@ public class TestDFSPropertiesConfiguration {
   @Test
   public void testParsing() {
     DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs, new Path(dfsBasePath + "/t1.props"));
-    TypedProperties props = cfg.getConfig();
+    TypedProperties props = cfg.getProps();
     assertEquals(5, props.size());
     assertThrows(IllegalArgumentException.class, () -> {
       props.getString("invalid.key");
@@ -119,7 +132,7 @@ public class TestDFSPropertiesConfiguration {
   @Test
   public void testIncludes() {
     DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs, new Path(dfsBasePath + "/t3.props"));
-    TypedProperties props = cfg.getConfig();
+    TypedProperties props = cfg.getProps();
 
     assertEquals(123, props.getInteger("int.prop"));
     assertEquals(243.4, props.getDouble("double.prop"), 0.001);
@@ -127,7 +140,30 @@ public class TestDFSPropertiesConfiguration {
     assertEquals("t3.value", props.getString("string.prop"));
     assertEquals(1354354354, props.getLong("long.prop"));
     assertThrows(IllegalStateException.class, () -> {
-      new DFSPropertiesConfiguration(dfs, new Path(dfsBasePath + "/t4.props"));
+      cfg.addPropsFromFile(new Path(dfsBasePath + "/t4.props"));
     }, "Should error out on a self-included file.");
   }
+
+  @Test
+  public void testNoGlobalConfFileConfigured() {
+    ENVIRONMENT_VARIABLES.clear(DFSPropertiesConfiguration.CONF_FILE_DIR_ENV_NAME);
+    // Should not throw any exception when no external configuration file configured
+    DFSPropertiesConfiguration.refreshGlobalProps();
+    assertEquals(0, DFSPropertiesConfiguration.getGlobalProps().size());
+  }
+
+  @Test
+  public void testLoadGlobalConfFile() {
+    // set HUDI_CONF_DIR
+    String testPropsFilePath = new File("src/test/resources/external-config").getAbsolutePath();
+    ENVIRONMENT_VARIABLES.set(DFSPropertiesConfiguration.CONF_FILE_DIR_ENV_NAME, testPropsFilePath);
+
+    DFSPropertiesConfiguration.refreshGlobalProps();
+    assertEquals(5, DFSPropertiesConfiguration.getGlobalProps().size());
+    assertEquals("jdbc:hive2://localhost:10000", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.datasource.hive_sync.jdbcurl"));
+    assertEquals("true", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.datasource.hive_sync.use_jdbc"));
+    assertEquals("false", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.datasource.hive_sync.support_timestamp"));
+    assertEquals("BLOOM", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.index.type"));
+    assertEquals("true", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.metadata.enable"));
+  }
 }

hudi-common/src/test/resources/external-config/hudi-defaults.conf

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default system properties included when running Hudi jobs.
+# This is useful for setting default environmental settings.
+
+# Example:
+hoodie.datasource.hive_sync.jdbcurl            jdbc:hive2://localhost:10000
+hoodie.datasource.hive_sync.use_jdbc           true
+hoodie.datasource.hive_sync.support_timestamp  false
+hoodie.index.type                              BLOOM
+hoodie.metadata.enable                         true