lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Ensure TableMetaClient and FileSystem instances have exclusive copy of Configuration

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-06-20 08:34:56 -07:00
committed by Balaji Varadarajan
parent a0d7ab2384
commit 2c40e8419e
5 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java
View File

@@ -68,7 +68,7 @@ public class EmbeddedTimelineService {
}
public void startServer() throws IOException {
server = new TimelineService(0, viewManager);
server = new TimelineService(0, viewManager, hadoopConf.get());
serverPort = server.startService();
logger.info("Started embedded timeline server at " + hostAddr + ":" + serverPort);
}

8
hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java
View File

@@ -29,11 +29,15 @@ public class SerializableConfiguration implements Serializable {
private transient Configuration configuration;
public SerializableConfiguration(Configuration configuration) {
this.configuration = configuration;
this.configuration = new Configuration(configuration);
}
public SerializableConfiguration(SerializableConfiguration configuration) {
this.configuration = configuration.get();
}
public Configuration get() {
return configuration;
return new Configuration(configuration);
}
private void writeObject(ObjectOutputStream out) throws IOException {

2
hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java
View File

@@ -85,7 +85,7 @@ public class HoodieTableMetaClient implements Serializable {
throws DatasetNotFoundException {
log.info("Loading HoodieTableMetaClient from " + basePath);
this.basePath = basePath;
this.hadoopConf = new SerializableConfiguration(new Configuration(conf));
this.hadoopConf = new SerializableConfiguration(conf);
Path basePathDir = new Path(this.basePath);
this.metaPath = basePath + File.separator + METAFOLDER_NAME;
Path metaPathDir = new Path(this.metaPath);

2
hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java
View File

@@ -65,7 +65,7 @@ public class FileSystemViewManager {
public FileSystemViewManager(SerializableConfiguration conf, FileSystemViewStorageConfig viewStorageConfig,
Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) {
this.conf = conf;
this.conf = new SerializableConfiguration(conf);
this.viewStorageConfig = viewStorageConfig;
this.globalViewMap = new ConcurrentHashMap<>();
this.viewCreator = viewCreator;

9
hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java
View File

@@ -50,13 +50,18 @@ public class TimelineService {
return serverPort;
}
public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager) throws IOException {
this.conf = FSUtils.prepareHadoopConf(new Configuration());
public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager,
Configuration conf) throws IOException {
this.conf = FSUtils.prepareHadoopConf(conf);
this.fs = FileSystem.get(conf);
this.serverPort = serverPort;
this.fsViewsManager = globalFileSystemViewManager;
}
public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager) throws IOException {
this(serverPort, globalFileSystemViewManager, new Configuration());
}
public TimelineService(Config config) throws IOException {
this(config.serverPort, buildFileSystemViewManager(config,
new SerializableConfiguration(FSUtils.prepareHadoopConf(new Configuration()))));