lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2779] Cache BaseDir if HudiTableNotFound Exception thrown (#4014)

Browse Source
This commit is contained in:
RexAn
2021-12-09 18:34:11 +08:00
committed by GitHub
parent 5ac9ce7289
commit f612a20815
2 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -71,7 +71,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
/**
* Paths that are known to be non-hoodie tables.
*/
private Set<String> nonHoodiePathCache;
Set<String> nonHoodiePathCache;
/**
* Table Meta Client Cache.
@@ -167,6 +167,13 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
}
if (baseDir != null) {
// Check whether baseDir in nonHoodiePathCache
if (nonHoodiePathCache.contains(baseDir.toString())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Accepting non-hoodie path from cache: " + path);
}
return true;
}
HoodieTableFileSystemView fsView = null;
try {
HoodieTableMetaClient metaClient = metaClientCache.get(baseDir.toString());
@@ -198,9 +205,10 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
} catch (TableNotFoundException e) {
// Non-hoodie path, accept it.
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("(1) Caching non-hoodie path under %s \n", folder.toString()));
LOG.debug(String.format("(1) Caching non-hoodie path under %s with basePath %s \n", folder.toString(), baseDir.toString()));
}
nonHoodiePathCache.add(folder.toString());
nonHoodiePathCache.add(baseDir.toString());
return true;
} finally {
if (fsView != null) {

4
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java
View File

@@ -68,9 +68,11 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
assertFalse(pathFilter.accept(testTable.getInflightCommitFilePath("003")));
assertFalse(pathFilter.accept(testTable.getRequestedCompactionFilePath("004")));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/")));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/hoodie.properties")));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME)));
assertEquals(1, pathFilter.metaClientCache.size());
assertEquals(0, pathFilter.nonHoodiePathCache.size(), "NonHoodiePathCache size should be 0");
}
@Test
@@ -82,6 +84,7 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
java.nio.file.Path path2 = Paths.get(basePath, "nonhoodiefolder/somefile");
Files.createFile(path2);
assertTrue(pathFilter.accept(new Path(path2.toUri())));
assertEquals(2, pathFilter.nonHoodiePathCache.size(), "NonHoodiePathCache size should be 2");
}
@Test
@@ -93,5 +96,6 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
Path partitionPath2 = testTable.getPartitionPath(p2).getParent();
assertTrue(pathFilter.accept(partitionPath1), "Directories should be accepted");
assertTrue(pathFilter.accept(partitionPath2), "Directories should be accepted");
assertEquals(2, pathFilter.nonHoodiePathCache.size(), "NonHoodiePathCache size should be 2");
}
}