lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-539] Make ROPathFilter conf member serializable (#1415)

Browse Source
This commit is contained in:
vinoth chandar
2020-03-17 12:52:48 -07:00
committed by GitHub
parent f1d7bb381d
commit e3019031d8
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -18,6 +18,7 @@
package org.apache.hudi.hadoop;
import org.apache.hudi.common.SerializableConfiguration;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -66,7 +67,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
/**
* Hadoop configurations for the FileSystem.
*/
private Configuration conf;
private SerializableConfiguration conf;
private transient FileSystem fs;
@@ -77,7 +78,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
public HoodieROTablePathFilter(Configuration conf) {
this.hoodiePathCache = new HashMap<>();
this.nonHoodiePathCache = new HashSet<>();
this.conf = conf;
this.conf = new SerializableConfiguration(conf);
}
/**
@@ -102,7 +103,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
Path folder = null;
try {
if (fs == null) {
fs = path.getFileSystem(conf);
fs = path.getFileSystem(conf.get());
}
// Assumes path is a file