lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2779] Cache BaseDir if HudiTableNotFound Exception thrown (#4014)

Browse Source
This commit is contained in:
RexAn
2021-12-09 18:34:11 +08:00
committed by GitHub
parent 5ac9ce7289
commit f612a20815
2 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -71,7 +71,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
/**
* Paths that are known to be non-hoodie tables.
*/
private Set<String> nonHoodiePathCache;
Set<String> nonHoodiePathCache;
/**
* Table Meta Client Cache.
@@ -167,6 +167,13 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
}
if (baseDir != null) {
// Check whether baseDir in nonHoodiePathCache
if (nonHoodiePathCache.contains(baseDir.toString())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Accepting non-hoodie path from cache: " + path);
}
return true;
}
HoodieTableFileSystemView fsView = null;
try {
HoodieTableMetaClient metaClient = metaClientCache.get(baseDir.toString());
@@ -198,9 +205,10 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
} catch (TableNotFoundException e) {
// Non-hoodie path, accept it.
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("(1) Caching non-hoodie path under %s \n", folder.toString()));
LOG.debug(String.format("(1) Caching non-hoodie path under %s with basePath %s \n", folder.toString(), baseDir.toString()));
}
nonHoodiePathCache.add(folder.toString());
nonHoodiePathCache.add(baseDir.toString());
return true;
} finally {
if (fsView != null) {