lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2362] Add external config file support (#3416)

Browse Source 
Co-authored-by: Wenning Ding <wenningd@amazon.com>
This commit is contained in:
wenningd
2021-11-18 01:59:26 -08:00
committed by GitHub
parent 8772cec4bd
commit 24def0b30d
25 changed files with 426 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
View File

@@ -18,14 +18,20 @@
package org.apache.hudi.common.config;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.HashSet;
import java.util.Set;
@@ -43,72 +49,110 @@ public class DFSPropertiesConfiguration {
private static final Logger LOG = LogManager.getLogger(DFSPropertiesConfiguration.class);
public static final String DEFAULT_PROPERTIES_FILE = "hudi-defaults.conf";
public static final String CONF_FILE_DIR_ENV_NAME = "HUDI_CONF_DIR";
public static final String DEFAULT_CONF_FILE_DIR = "file:/etc/hudi/conf";
// props read from hudi-defaults.conf
private static TypedProperties GLOBAL_PROPS = loadGlobalProps();
private final FileSystem fs;
private final Path rootFile;
private Path currentFilePath;
private final TypedProperties props;
// props read from user defined configuration file or input stream
private final HoodieConfig hoodieConfig;
// Keep track of files visited, to detect loops
private final Set<String> visitedFiles;
private final Set<String> visitedFilePaths;
public DFSPropertiesConfiguration(FileSystem fs, Path rootFile, TypedProperties defaults) {
public DFSPropertiesConfiguration(FileSystem fs, Path filePath) {
this.fs = fs;
this.rootFile = rootFile;
this.props = defaults;
this.visitedFiles = new HashSet<>();
visitFile(rootFile);
}
public DFSPropertiesConfiguration(FileSystem fs, Path rootFile) {
this(fs, rootFile, new TypedProperties());
this.currentFilePath = filePath;
this.hoodieConfig = new HoodieConfig();
this.visitedFilePaths = new HashSet<>();
addPropsFromFile(filePath);
}
public DFSPropertiesConfiguration() {
this.fs = null;
this.rootFile = null;
this.props = new TypedProperties();
this.visitedFiles = new HashSet<>();
this.currentFilePath = null;
this.hoodieConfig = new HoodieConfig();
this.visitedFilePaths = new HashSet<>();
}
private String[] splitProperty(String line) {
int ind = line.indexOf('=');
String k = line.substring(0, ind).trim();
String v = line.substring(ind + 1).trim();
return new String[] {k, v};
}
private void visitFile(Path file) {
try {
if (visitedFiles.contains(file.getName())) {
throw new IllegalStateException("Loop detected; file " + file + " already referenced");
/**
* Load global props from hudi-defaults.conf which is under CONF_FILE_DIR_ENV_NAME.
* @return Typed Properties
*/
public static TypedProperties loadGlobalProps() {
DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration();
Option<Path> defaultConfPath = getConfPathFromEnv();
if (defaultConfPath.isPresent()) {
conf.addPropsFromFile(defaultConfPath.get());
} else {
try {
conf.addPropsFromFile(new Path(DEFAULT_CONF_FILE_DIR));
} catch (Exception ignored) {
LOG.debug("Didn't find config file under default conf file dir: " + DEFAULT_CONF_FILE_DIR);
}
visitedFiles.add(file.getName());
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(file)));
addProperties(reader);
}
return conf.getProps();
}
public static void refreshGlobalProps() {
GLOBAL_PROPS = loadGlobalProps();
}
public static void clearGlobalProps() {
GLOBAL_PROPS = new TypedProperties();
}
/**
* Add properties from external configuration files.
*
* @param filePath File path for configuration file
*/
public void addPropsFromFile(Path filePath) {
if (visitedFilePaths.contains(filePath.toString())) {
throw new IllegalStateException("Loop detected; file " + filePath + " already referenced");
}
FileSystem fileSystem;
try {
fileSystem = fs != null ? fs : filePath.getFileSystem(new Configuration());
} catch (IOException e) {
throw new IllegalArgumentException("Cannot get the file system from file path", e);
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(fileSystem.open(filePath)))) {
visitedFilePaths.add(filePath.toString());
currentFilePath = filePath;
addPropsFromStream(reader);
} catch (IOException ioe) {
LOG.error("Error reading in properies from dfs", ioe);
LOG.error("Error reading in properties from dfs");
throw new IllegalArgumentException("Cannot read properties from dfs", ioe);
}
}
/**
* Add properties from input stream.
*
* Add properties from buffered reader.
*
* @param reader Buffered Reader
* @throws IOException
*/
public void addProperties(BufferedReader reader) throws IOException {
public void addPropsFromStream(BufferedReader reader) throws IOException {
try {
reader.lines().forEach(line -> {
if (line.startsWith("#") || line.equals("") || !line.contains("=")) {
if (!isValidLine(line)) {
return;
}
String[] split = splitProperty(line);
if (line.startsWith("include=") || line.startsWith("include =")) {
visitFile(new Path(rootFile.getParent(), split[1]));
Path includeFilePath = new Path(currentFilePath.getParent(), split[1]);
addPropsFromFile(includeFilePath);
} else {
props.setProperty(split[0], split[1]);
hoodieConfig.setValue(split[0], split[1]);
}
});
@@ -117,7 +161,46 @@ public class DFSPropertiesConfiguration {
}
}
public TypedProperties getConfig() {
return props;
public static TypedProperties getGlobalProps() {
final TypedProperties globalProps = new TypedProperties();
globalProps.putAll(GLOBAL_PROPS);
return globalProps;
}
public TypedProperties getProps() {
return new TypedProperties(hoodieConfig.getProps());
}
public TypedProperties getProps(boolean includeGlobalProps) {
return new TypedProperties(hoodieConfig.getProps(includeGlobalProps));
}
private static Option<Path> getConfPathFromEnv() {
String confDir = System.getenv(CONF_FILE_DIR_ENV_NAME);
if (confDir == null) {
LOG.warn("Cannot find " + CONF_FILE_DIR_ENV_NAME + ", please set it as the dir of " + DEFAULT_PROPERTIES_FILE);
return Option.empty();
}
if (StringUtils.isNullOrEmpty(URI.create(confDir).getScheme())) {
confDir = "file://" + confDir;
}
return Option.of(new Path(confDir + File.separator + DEFAULT_PROPERTIES_FILE));
}
private String[] splitProperty(String line) {
line = line.replaceAll("\\s+"," ");
String delimiter = line.contains("=") ? "=" : " ";
int ind = line.indexOf(delimiter);
String k = line.substring(0, ind).trim();
String v = line.substring(ind + 1).trim();
return new String[] {k, v};
}
private boolean isValidLine(String line) {
ValidationUtils.checkArgument(line != null, "passed line is null");
if (line.startsWith("#") || line.equals("")) {
return false;
}
return line.contains("=") || line.matches(".*\\s.*");
}
}

20
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
View File

@@ -58,6 +58,14 @@ public class HoodieConfig implements Serializable {
props.setProperty(cfg.key(), val);
}
public <T> void setValue(String key, String val) {
props.setProperty(key, val);
}
public void setAll(Properties properties) {
props.putAll(properties);
}
public <T> void setDefaultValue(ConfigProperty<T> configProperty) {
if (!contains(configProperty)) {
Option<T> inferValue = Option.empty();
@@ -167,7 +175,17 @@ public class HoodieConfig implements Serializable {
}
public Properties getProps() {
return props;
return getProps(false);
}
public Properties getProps(boolean includeGlobalProps) {
if (includeGlobalProps) {
Properties mergedProps = DFSPropertiesConfiguration.getGlobalProps();
mergedProps.putAll(props);
return mergedProps;
} else {
return props;
}
}
public void setDefaultOnCondition(boolean condition, HoodieConfig config) {