lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Ensure TableMetaClient and FileSystem instances have exclusive copy of Configuration

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-06-20 08:34:56 -07:00
committed by Balaji Varadarajan
parent a0d7ab2384
commit 2c40e8419e
5 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java
View File

@@ -68,7 +68,7 @@ public class EmbeddedTimelineService {
} }
public void startServer() throws IOException { public void startServer() throws IOException {
server = new TimelineService(0, viewManager); server = new TimelineService(0, viewManager, hadoopConf.get());
serverPort = server.startService(); serverPort = server.startService();
logger.info("Started embedded timeline server at " + hostAddr + ":" + serverPort); logger.info("Started embedded timeline server at " + hostAddr + ":" + serverPort);
} }

8
hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java
View File

@@ -29,11 +29,15 @@ public class SerializableConfiguration implements Serializable {
private transient Configuration configuration; private transient Configuration configuration;
public SerializableConfiguration(Configuration configuration) { public SerializableConfiguration(Configuration configuration) {
this.configuration = configuration; this.configuration = new Configuration(configuration);
}
public SerializableConfiguration(SerializableConfiguration configuration) {
this.configuration = configuration.get();
} }
public Configuration get() { public Configuration get() {
return configuration; return new Configuration(configuration);
} }
private void writeObject(ObjectOutputStream out) throws IOException { private void writeObject(ObjectOutputStream out) throws IOException {

2
hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java
View File

@@ -85,7 +85,7 @@ public class HoodieTableMetaClient implements Serializable {
throws DatasetNotFoundException { throws DatasetNotFoundException {
log.info("Loading HoodieTableMetaClient from " + basePath); log.info("Loading HoodieTableMetaClient from " + basePath);
this.basePath = basePath; this.basePath = basePath;
this.hadoopConf = new SerializableConfiguration(new Configuration(conf)); this.hadoopConf = new SerializableConfiguration(conf);
Path basePathDir = new Path(this.basePath); Path basePathDir = new Path(this.basePath);
this.metaPath = basePath + File.separator + METAFOLDER_NAME; this.metaPath = basePath + File.separator + METAFOLDER_NAME;
Path metaPathDir = new Path(this.metaPath); Path metaPathDir = new Path(this.metaPath);

2
hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java
View File

@@ -65,7 +65,7 @@ public class FileSystemViewManager {
public FileSystemViewManager(SerializableConfiguration conf, FileSystemViewStorageConfig viewStorageConfig, public FileSystemViewManager(SerializableConfiguration conf, FileSystemViewStorageConfig viewStorageConfig,
Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) { Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) {
this.conf = conf; this.conf = new SerializableConfiguration(conf);
this.viewStorageConfig = viewStorageConfig; this.viewStorageConfig = viewStorageConfig;
this.globalViewMap = new ConcurrentHashMap<>(); this.globalViewMap = new ConcurrentHashMap<>();
this.viewCreator = viewCreator; this.viewCreator = viewCreator;

9
hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java
View File

@@ -50,13 +50,18 @@ public class TimelineService {
return serverPort; return serverPort;
} }
public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager) throws IOException { public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager,
this.conf = FSUtils.prepareHadoopConf(new Configuration()); Configuration conf) throws IOException {
this.conf = FSUtils.prepareHadoopConf(conf);
this.fs = FileSystem.get(conf); this.fs = FileSystem.get(conf);
this.serverPort = serverPort; this.serverPort = serverPort;
this.fsViewsManager = globalFileSystemViewManager; this.fsViewsManager = globalFileSystemViewManager;
} }
public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager) throws IOException {
this(serverPort, globalFileSystemViewManager, new Configuration());
}
public TimelineService(Config config) throws IOException { public TimelineService(Config config) throws IOException {
this(config.serverPort, buildFileSystemViewManager(config, this(config.serverPort, buildFileSystemViewManager(config,
new SerializableConfiguration(FSUtils.prepareHadoopConf(new Configuration())))); new SerializableConfiguration(FSUtils.prepareHadoopConf(new Configuration()))));