[HUDI-1135] Make timeline server timeout settings configurable.
This commit is contained in:
@@ -169,9 +169,10 @@ public class FileSystemViewManager {
|
|||||||
private static RemoteHoodieTableFileSystemView createRemoteFileSystemView(SerializableConfiguration conf,
|
private static RemoteHoodieTableFileSystemView createRemoteFileSystemView(SerializableConfiguration conf,
|
||||||
FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
|
FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
|
||||||
LOG.info("Creating remote view for basePath " + metaClient.getBasePath() + ". Server="
|
LOG.info("Creating remote view for basePath " + metaClient.getBasePath() + ". Server="
|
||||||
+ viewConf.getRemoteViewServerHost() + ":" + viewConf.getRemoteViewServerPort());
|
+ viewConf.getRemoteViewServerHost() + ":" + viewConf.getRemoteViewServerPort() + ", Timeout="
|
||||||
|
+ viewConf.getRemoteTimelineClientTimeoutSecs());
|
||||||
return new RemoteHoodieTableFileSystemView(viewConf.getRemoteViewServerHost(), viewConf.getRemoteViewServerPort(),
|
return new RemoteHoodieTableFileSystemView(viewConf.getRemoteViewServerHost(), viewConf.getRemoteViewServerPort(),
|
||||||
metaClient);
|
metaClient, viewConf.getRemoteTimelineClientTimeoutSecs());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -44,6 +44,8 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig {
|
|||||||
public static final String FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION =
|
public static final String FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION =
|
||||||
"hoodie.filesystem.view.spillable.bootstrap.base.file.mem.fraction";
|
"hoodie.filesystem.view.spillable.bootstrap.base.file.mem.fraction";
|
||||||
private static final String ROCKSDB_BASE_PATH_PROP = "hoodie.filesystem.view.rocksdb.base.path";
|
private static final String ROCKSDB_BASE_PATH_PROP = "hoodie.filesystem.view.rocksdb.base.path";
|
||||||
|
public static final String FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS =
|
||||||
|
"hoodie.filesystem.view.remote.timeout.secs";
|
||||||
|
|
||||||
public static final FileSystemViewStorageType DEFAULT_VIEW_STORAGE_TYPE = FileSystemViewStorageType.MEMORY;
|
public static final FileSystemViewStorageType DEFAULT_VIEW_STORAGE_TYPE = FileSystemViewStorageType.MEMORY;
|
||||||
public static final FileSystemViewStorageType DEFAULT_SECONDARY_VIEW_STORAGE_TYPE = FileSystemViewStorageType.MEMORY;
|
public static final FileSystemViewStorageType DEFAULT_SECONDARY_VIEW_STORAGE_TYPE = FileSystemViewStorageType.MEMORY;
|
||||||
@@ -52,7 +54,7 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig {
|
|||||||
public static final String DEFAULT_FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE = "false";
|
public static final String DEFAULT_FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE = "false";
|
||||||
public static final String DEFUALT_REMOTE_VIEW_SERVER_HOST = "localhost";
|
public static final String DEFUALT_REMOTE_VIEW_SERVER_HOST = "localhost";
|
||||||
public static final Integer DEFAULT_REMOTE_VIEW_SERVER_PORT = 26754;
|
public static final Integer DEFAULT_REMOTE_VIEW_SERVER_PORT = 26754;
|
||||||
|
public static final Integer DEFAULT_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS = 5 * 60; // 5 min
|
||||||
public static final String DEFAULT_VIEW_SPILLABLE_DIR = "/tmp/view_map/";
|
public static final String DEFAULT_VIEW_SPILLABLE_DIR = "/tmp/view_map/";
|
||||||
private static final Double DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION = 0.01;
|
private static final Double DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION = 0.01;
|
||||||
private static final Double DEFAULT_MEM_FRACTION_FOR_EXTERNAL_DATA_FILE = 0.05;
|
private static final Double DEFAULT_MEM_FRACTION_FOR_EXTERNAL_DATA_FILE = 0.05;
|
||||||
@@ -91,6 +93,10 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig {
|
|||||||
return Integer.parseInt(props.getProperty(FILESYSTEM_VIEW_REMOTE_PORT));
|
return Integer.parseInt(props.getProperty(FILESYSTEM_VIEW_REMOTE_PORT));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Integer getRemoteTimelineClientTimeoutSecs() {
|
||||||
|
return Integer.parseInt(props.getProperty(FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS));
|
||||||
|
}
|
||||||
|
|
||||||
public long getMaxMemoryForFileGroupMap() {
|
public long getMaxMemoryForFileGroupMap() {
|
||||||
long totalMemory = Long.parseLong(props.getProperty(FILESYSTEM_VIEW_SPILLABLE_MEM));
|
long totalMemory = Long.parseLong(props.getProperty(FILESYSTEM_VIEW_SPILLABLE_MEM));
|
||||||
return totalMemory - getMaxMemoryForPendingCompaction() - getMaxMemoryForBootstrapBaseFile();
|
return totalMemory - getMaxMemoryForPendingCompaction() - getMaxMemoryForBootstrapBaseFile();
|
||||||
@@ -175,6 +181,11 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig {
|
|||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Builder withRemoteTimelineClientTimeoutSecs(Long timelineClientTimeoutSecs) {
|
||||||
|
props.setProperty(FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS, timelineClientTimeoutSecs.toString());
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
public Builder withMemFractionForPendingCompaction(Double memFractionForPendingCompaction) {
|
public Builder withMemFractionForPendingCompaction(Double memFractionForPendingCompaction) {
|
||||||
props.setProperty(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION, memFractionForPendingCompaction.toString());
|
props.setProperty(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION, memFractionForPendingCompaction.toString());
|
||||||
return this;
|
return this;
|
||||||
@@ -216,6 +227,8 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig {
|
|||||||
DEFAULT_VIEW_SPILLABLE_DIR);
|
DEFAULT_VIEW_SPILLABLE_DIR);
|
||||||
setDefaultOnCondition(props, !props.containsKey(FILESYSTEM_VIEW_SPILLABLE_MEM), FILESYSTEM_VIEW_SPILLABLE_MEM,
|
setDefaultOnCondition(props, !props.containsKey(FILESYSTEM_VIEW_SPILLABLE_MEM), FILESYSTEM_VIEW_SPILLABLE_MEM,
|
||||||
DEFAULT_MAX_MEMORY_FOR_VIEW.toString());
|
DEFAULT_MAX_MEMORY_FOR_VIEW.toString());
|
||||||
|
setDefaultOnCondition(props, !props.containsKey(FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS),
|
||||||
|
FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS, DEFAULT_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS.toString());
|
||||||
setDefaultOnCondition(props, !props.containsKey(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION),
|
setDefaultOnCondition(props, !props.containsKey(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION),
|
||||||
FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION, DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION.toString());
|
FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION, DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION.toString());
|
||||||
setDefaultOnCondition(props, !props.containsKey(FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION),
|
setDefaultOnCondition(props, !props.containsKey(FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION),
|
||||||
|
|||||||
@@ -115,6 +115,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
|
|||||||
private final HoodieTableMetaClient metaClient;
|
private final HoodieTableMetaClient metaClient;
|
||||||
private final HoodieTimeline timeline;
|
private final HoodieTimeline timeline;
|
||||||
private final ObjectMapper mapper;
|
private final ObjectMapper mapper;
|
||||||
|
private final int timeoutSecs;
|
||||||
|
|
||||||
private boolean closed = false;
|
private boolean closed = false;
|
||||||
|
|
||||||
@@ -123,12 +124,17 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
|
|||||||
}
|
}
|
||||||
|
|
||||||
public RemoteHoodieTableFileSystemView(String server, int port, HoodieTableMetaClient metaClient) {
|
public RemoteHoodieTableFileSystemView(String server, int port, HoodieTableMetaClient metaClient) {
|
||||||
|
this(server, port, metaClient, 300);
|
||||||
|
}
|
||||||
|
|
||||||
|
public RemoteHoodieTableFileSystemView(String server, int port, HoodieTableMetaClient metaClient, int timeoutSecs) {
|
||||||
this.basePath = metaClient.getBasePath();
|
this.basePath = metaClient.getBasePath();
|
||||||
this.serverHost = server;
|
this.serverHost = server;
|
||||||
this.serverPort = port;
|
this.serverPort = port;
|
||||||
this.mapper = new ObjectMapper();
|
this.mapper = new ObjectMapper();
|
||||||
this.metaClient = metaClient;
|
this.metaClient = metaClient;
|
||||||
this.timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
|
this.timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
|
||||||
|
this.timeoutSecs = timeoutSecs;
|
||||||
}
|
}
|
||||||
|
|
||||||
private <T> T executeRequest(String requestPath, Map<String, String> queryParameters, TypeReference reference,
|
private <T> T executeRequest(String requestPath, Map<String, String> queryParameters, TypeReference reference,
|
||||||
@@ -147,7 +153,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
|
|||||||
String url = builder.toString();
|
String url = builder.toString();
|
||||||
LOG.info("Sending request : (" + url + ")");
|
LOG.info("Sending request : (" + url + ")");
|
||||||
Response response;
|
Response response;
|
||||||
int timeout = 1000 * 300; // 5 min timeout
|
int timeout = this.timeoutSecs * 1000; // msec
|
||||||
switch (method) {
|
switch (method) {
|
||||||
case GET:
|
case GET:
|
||||||
response = Request.Get(url).connectTimeout(timeout).socketTimeout(timeout).execute();
|
response = Request.Get(url).connectTimeout(timeout).socketTimeout(timeout).execute();
|
||||||
|
|||||||
Reference in New Issue
Block a user