lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Ensure Hoodie metadata folder and files are filtered out when constructing Parquet Data Source

Browse Source
This commit is contained in:
Balaji Varadarajan
2018-09-28 21:41:28 -07:00
committed by vinoth chandar
parent 06bdba3cef
commit 9710b5a3a6
3 changed files with 35 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieROTablePathFilter.java
View File

@@ -111,6 +111,16 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
return hoodiePathCache.get(folder.toString()).contains(path);
}
// Skip all files that are descendants of .hoodie in its path.
String filePath = path.toString();
if (filePath.contains("/" + HoodieTableMetaClient.METAFOLDER_NAME + "/")
|| filePath.endsWith("/" + HoodieTableMetaClient.METAFOLDER_NAME)) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Skipping Hoodie Metadata file %s \n", filePath));
}
return false;
}
// Perform actual checking.
Path baseDir;
if (HoodiePartitionMetadata.hasPartitionMetadata(fs, folder)) {