lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Add constructor to HoodieROTablePathFilter (#1413)

Browse Source 
Allows HoodieROTablePathFilter to accept a configuration for
initializing the filesystem. This fixes a bug with Presto's use of this
pathfilter.

Co-authored-by: Brandon Scheller <bschelle@amazon.com>
This commit is contained in:
bschell
2020-03-16 15:19:16 -07:00
committed by GitHub
parent 3ef9e885ca
commit 418f9bb2e9
1 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -63,12 +63,21 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
*/ */
private HashSet<String> nonHoodiePathCache; private HashSet<String> nonHoodiePathCache;
/**
* Hadoop configurations for the FileSystem.
*/
private Configuration conf;
private transient FileSystem fs; private transient FileSystem fs;
public HoodieROTablePathFilter() { public HoodieROTablePathFilter() {
hoodiePathCache = new HashMap<>(); this(new Configuration());
nonHoodiePathCache = new HashSet<>(); }
public HoodieROTablePathFilter(Configuration conf) {
this.hoodiePathCache = new HashMap<>();
this.nonHoodiePathCache = new HashSet<>();
this.conf = conf;
} }
/** /**
@@ -93,7 +102,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
Path folder = null; Path folder = null;
try { try {
if (fs == null) { if (fs == null) {
fs = path.getFileSystem(new Configuration()); fs = path.getFileSystem(conf);
} }
// Assumes path is a file // Assumes path is a file