lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2779] Cache BaseDir if HudiTableNotFound Exception thrown (#4014)

Browse Source
This commit is contained in:
RexAn
2021-12-09 18:34:11 +08:00
committed by GitHub
parent 5ac9ce7289
commit f612a20815
2 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -71,7 +71,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
/** /**
* Paths that are known to be non-hoodie tables. * Paths that are known to be non-hoodie tables.
*/ */
private Set<String> nonHoodiePathCache; Set<String> nonHoodiePathCache;
/** /**
* Table Meta Client Cache. * Table Meta Client Cache.
@@ -167,6 +167,13 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
} }
if (baseDir != null) { if (baseDir != null) {
// Check whether baseDir in nonHoodiePathCache
if (nonHoodiePathCache.contains(baseDir.toString())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Accepting non-hoodie path from cache: " + path);
}
return true;
}
HoodieTableFileSystemView fsView = null; HoodieTableFileSystemView fsView = null;
try { try {
HoodieTableMetaClient metaClient = metaClientCache.get(baseDir.toString()); HoodieTableMetaClient metaClient = metaClientCache.get(baseDir.toString());
@@ -198,9 +205,10 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
} catch (TableNotFoundException e) { } catch (TableNotFoundException e) {
// Non-hoodie path, accept it. // Non-hoodie path, accept it.
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug(String.format("(1) Caching non-hoodie path under %s \n", folder.toString())); LOG.debug(String.format("(1) Caching non-hoodie path under %s with basePath %s \n", folder.toString(), baseDir.toString()));
} }
nonHoodiePathCache.add(folder.toString()); nonHoodiePathCache.add(folder.toString());
nonHoodiePathCache.add(baseDir.toString());
return true; return true;
} finally { } finally {
if (fsView != null) { if (fsView != null) {

4
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java
View File

@@ -68,9 +68,11 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
assertFalse(pathFilter.accept(testTable.getInflightCommitFilePath("003"))); assertFalse(pathFilter.accept(testTable.getInflightCommitFilePath("003")));
assertFalse(pathFilter.accept(testTable.getRequestedCompactionFilePath("004"))); assertFalse(pathFilter.accept(testTable.getRequestedCompactionFilePath("004")));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"))); assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/")));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/hoodie.properties")));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME))); assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME)));
assertEquals(1, pathFilter.metaClientCache.size()); assertEquals(1, pathFilter.metaClientCache.size());
assertEquals(0, pathFilter.nonHoodiePathCache.size(), "NonHoodiePathCache size should be 0");
} }
@Test @Test
@@ -82,6 +84,7 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
java.nio.file.Path path2 = Paths.get(basePath, "nonhoodiefolder/somefile"); java.nio.file.Path path2 = Paths.get(basePath, "nonhoodiefolder/somefile");
Files.createFile(path2); Files.createFile(path2);
assertTrue(pathFilter.accept(new Path(path2.toUri()))); assertTrue(pathFilter.accept(new Path(path2.toUri())));
assertEquals(2, pathFilter.nonHoodiePathCache.size(), "NonHoodiePathCache size should be 2");
} }
@Test @Test
@@ -93,5 +96,6 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
Path partitionPath2 = testTable.getPartitionPath(p2).getParent(); Path partitionPath2 = testTable.getPartitionPath(p2).getParent();
assertTrue(pathFilter.accept(partitionPath1), "Directories should be accepted"); assertTrue(pathFilter.accept(partitionPath1), "Directories should be accepted");
assertTrue(pathFilter.accept(partitionPath2), "Directories should be accepted"); assertTrue(pathFilter.accept(partitionPath2), "Directories should be accepted");
assertEquals(2, pathFilter.nonHoodiePathCache.size(), "NonHoodiePathCache size should be 2");
} }
} }