lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Revert "[HUDI-2489]Tuning HoodieROTablePathFilter by caching hoodieTableFileSystemView, aiming to reduce unnecessary list/get requests"

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
zhangyue19921010
2021-12-04 10:56:53 +08:00
committed by GitHub
parent 383d5edc16
commit 5616830ae1
1 changed files with 6 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -78,11 +78,6 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
*/ */
Map<String, HoodieTableMetaClient> metaClientCache; Map<String, HoodieTableMetaClient> metaClientCache;
/**
* HoodieTableFileSystemView Cache.
*/
private Map<String, HoodieTableFileSystemView> hoodieTableFileSystemViewCache;
/** /**
* Hadoop configurations for the FileSystem. * Hadoop configurations for the FileSystem.
*/ */
@@ -102,7 +97,6 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
this.nonHoodiePathCache = new HashSet<>(); this.nonHoodiePathCache = new HashSet<>();
this.conf = new SerializableConfiguration(conf); this.conf = new SerializableConfiguration(conf);
this.metaClientCache = new HashMap<>(); this.metaClientCache = new HashMap<>();
this.hoodieTableFileSystemViewCache = new HashMap<>();
} }
/** /**
@@ -181,15 +175,8 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
metaClientCache.put(baseDir.toString(), metaClient); metaClientCache.put(baseDir.toString(), metaClient);
} }
HoodieTableMetaClient finalMetaClient = metaClient; fsView = FileSystemViewManager.createInMemoryFileSystemView(engineContext,
fsView = hoodieTableFileSystemViewCache.computeIfAbsent(baseDir.toString(), key -> metaClient, HoodieInputFormatUtils.buildMetadataConfig(getConf()));
FileSystemViewManager.createInMemoryFileSystemView(
engineContext,
finalMetaClient,
HoodieInputFormatUtils.buildMetadataConfig(getConf())
)
);
String partition = FSUtils.getRelativePartitionPath(new Path(metaClient.getBasePath()), folder); String partition = FSUtils.getRelativePartitionPath(new Path(metaClient.getBasePath()), folder);
List<HoodieBaseFile> latestFiles = fsView.getLatestBaseFiles(partition).collect(Collectors.toList()); List<HoodieBaseFile> latestFiles = fsView.getLatestBaseFiles(partition).collect(Collectors.toList());
// populate the cache // populate the cache
@@ -215,6 +202,10 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
} }
nonHoodiePathCache.add(folder.toString()); nonHoodiePathCache.add(folder.toString());
return true; return true;
} finally {
if (fsView != null) {
fsView.close();
}
} }
} else { } else {
// files is at < 3 level depth in FS tree, can't be hoodie dataset // files is at < 3 level depth in FS tree, can't be hoodie dataset