lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2880] Fixing loading of props from default dir (#4167)

Browse Source 
* Fixing loading of props from default dir

* addressing comments
This commit is contained in:
Sivabalan Narayanan
2021-12-01 03:02:30 -05:00
committed by GitHub
parent 9b254b6fc5
commit f4c25ba3fd
2 changed files with 19 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
View File

@@ -18,18 +18,21 @@
package org.apache.hudi.common.config;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
@@ -53,10 +56,9 @@ public class DFSPropertiesConfiguration {
private static final Logger LOG = LogManager.getLogger(DFSPropertiesConfiguration.class);
public static final String DEFAULT_PROPERTIES_FILE = "hudi-defaults.conf";
public static final String CONF_FILE_DIR_ENV_NAME = "HUDI_CONF_DIR";
public static final String DEFAULT_CONF_FILE_DIR = "file:/etc/hudi/conf";
public static final Path DEFAULT_PATH = new Path(DEFAULT_CONF_FILE_DIR + "/" + DEFAULT_PROPERTIES_FILE);
// props read from hudi-defaults.conf
private static TypedProperties GLOBAL_PROPS = loadGlobalProps();
@@ -97,11 +99,7 @@ public class DFSPropertiesConfiguration {
if (defaultConfPath.isPresent()) {
conf.addPropsFromFile(defaultConfPath.get());
} else {
try {
conf.addPropsFromFile(new Path(DEFAULT_CONF_FILE_DIR));
} catch (Exception ignored) {
LOG.warn("Didn't find config file under default conf file dir: " + DEFAULT_CONF_FILE_DIR);
}
conf.addPropsFromFile(DEFAULT_PATH);
}
return conf.getProps();
}
@@ -128,14 +126,19 @@ public class DFSPropertiesConfiguration {
filePath.toString(),
Option.ofNullable(hadoopConfig).orElseGet(Configuration::new)
);
try {
if (filePath.equals(DEFAULT_PATH) && !fs.exists(filePath)) {
LOG.warn("Properties file " + filePath + " not found. Ignoring to load props file");
return;
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath)))) {
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath)));
visitedFilePaths.add(filePath.toString());
currentFilePath = filePath;
addPropsFromStream(reader);
} catch (IOException ioe) {
LOG.error("Error reading in properties from dfs");
throw new IllegalArgumentException("Cannot read properties from dfs", ioe);
LOG.error("Error reading in properties from dfs from file " + filePath);
throw new HoodieIOException("Cannot read properties from dfs from file " + filePath, ioe);
}
}
@@ -192,7 +195,7 @@ public class DFSPropertiesConfiguration {
}
private String[] splitProperty(String line) {
line = line.replaceAll("\\s+"," ");
line = line.replaceAll("\\s+", " ");
String delimiter = line.contains("=") ? "=" : " ";
int ind = line.indexOf(delimiter);
String k = line.substring(0, ind).trim();

5
hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
View File

@@ -145,7 +145,7 @@ public class TestDFSPropertiesConfiguration {
}
@Test
public void testLocalFileSystemLoading() {
public void testLocalFileSystemLoading() throws IOException {
DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/t1.props"));
cfg.addPropsFromFile(
@@ -156,8 +156,7 @@ public class TestDFSPropertiesConfiguration {
.getResource("props/test.properties")
.getPath()
)
)
);
));
TypedProperties props = cfg.getProps();