lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-539] Make ROPathFilter conf member serializable (#1415)

Browse Source
This commit is contained in:
vinoth chandar
2020-03-17 12:52:48 -07:00
committed by GitHub
parent f1d7bb381d
commit e3019031d8
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -18,6 +18,7 @@
package org.apache.hudi.hadoop; package org.apache.hudi.hadoop;
import org.apache.hudi.common.SerializableConfiguration;
import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -66,7 +67,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
/** /**
* Hadoop configurations for the FileSystem. * Hadoop configurations for the FileSystem.
*/ */
private Configuration conf; private SerializableConfiguration conf;
private transient FileSystem fs; private transient FileSystem fs;
@@ -77,7 +78,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
public HoodieROTablePathFilter(Configuration conf) { public HoodieROTablePathFilter(Configuration conf) {
this.hoodiePathCache = new HashMap<>(); this.hoodiePathCache = new HashMap<>();
this.nonHoodiePathCache = new HashSet<>(); this.nonHoodiePathCache = new HashSet<>();
this.conf = conf; this.conf = new SerializableConfiguration(conf);
} }
/** /**
@@ -102,7 +103,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
Path folder = null; Path folder = null;
try { try {
if (fs == null) { if (fs == null) {
fs = path.getFileSystem(conf); fs = path.getFileSystem(conf.get());
} }
// Assumes path is a file // Assumes path is a file