lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2671] Making error -> warn logs from timeline server with concurrent writers for inconsistent state (#4088)

Browse Source 
* Making error -> warn logs from timeline server with concurrent writers for inconsistent state

* Fixing bad request response exception for timeline out of sync

* Addressing feedback. removed write concurrency mode depedency
This commit is contained in:
Sivabalan Narayanan
2021-11-25 14:21:32 -05:00
committed by GitHub
parent 7bb90e8caf
commit f692078d32
3 changed files with 76 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java
View File

@@ -32,6 +32,8 @@ import org.apache.hudi.common.util.Functions.Function3;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpResponseException;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -65,7 +67,7 @@ public class PriorityBasedFileSystemView implements SyncableFileSystemView, Seri
try {
return preferredFunction.apply();
} catch (RuntimeException re) {
LOG.error("Got error running preferred function. Trying secondary", re);
handleRuntimeException(re);
errorOnPreferredView = true;
return secondaryFunction.apply();
}
@@ -80,7 +82,7 @@ public class PriorityBasedFileSystemView implements SyncableFileSystemView, Seri
try {
return preferredFunction.apply(val);
} catch (RuntimeException re) {
LOG.error("Got error running preferred function. Trying secondary", re);
handleRuntimeException(re);
errorOnPreferredView = true;
return secondaryFunction.apply(val);
}
@@ -96,7 +98,7 @@ public class PriorityBasedFileSystemView implements SyncableFileSystemView, Seri
try {
return preferredFunction.apply(val, val2);
} catch (RuntimeException re) {
LOG.error("Got error running preferred function. Trying secondary", re);
handleRuntimeException(re);
errorOnPreferredView = true;
return secondaryFunction.apply(val, val2);
}
@@ -112,13 +114,21 @@ public class PriorityBasedFileSystemView implements SyncableFileSystemView, Seri
try {
return preferredFunction.apply(val, val2, val3);
} catch (RuntimeException re) {
LOG.error("Got error running preferred function. Trying secondary", re);
handleRuntimeException(re);
errorOnPreferredView = true;
return secondaryFunction.apply(val, val2, val3);
}
}
}
private void handleRuntimeException(RuntimeException re) {
if (re.getCause() instanceof HttpResponseException && ((HttpResponseException)re.getCause()).getStatusCode() == HttpStatus.SC_BAD_REQUEST) {
LOG.warn("Got error running preferred function. Likely due to another concurrent writer in progress. Trying secondary");
} else {
LOG.error("Got error running preferred function. Trying secondary", re);
}
}
@Override
public Stream<HoodieBaseFile> getLatestBaseFiles(String partitionPath) {
return execute(partitionPath, preferredView::getLatestBaseFiles, secondaryView::getLatestBaseFiles);