lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Revert "[HUDI-2489]Tuning HoodieROTablePathFilter by caching hoodieTableFileSystemView, aiming to reduce unnecessary list/get requests"

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
zhangyue19921010
2021-12-04 10:56:53 +08:00
committed by GitHub
parent 383d5edc16
commit 5616830ae1
1 changed files with 6 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -78,11 +78,6 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
*/
Map<String, HoodieTableMetaClient> metaClientCache;
/**
* HoodieTableFileSystemView Cache.
*/
private Map<String, HoodieTableFileSystemView> hoodieTableFileSystemViewCache;
/**
* Hadoop configurations for the FileSystem.
*/
@@ -102,7 +97,6 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
this.nonHoodiePathCache = new HashSet<>();
this.conf = new SerializableConfiguration(conf);
this.metaClientCache = new HashMap<>();
this.hoodieTableFileSystemViewCache = new HashMap<>();
}
/**
@@ -181,15 +175,8 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
metaClientCache.put(baseDir.toString(), metaClient);
}
HoodieTableMetaClient finalMetaClient = metaClient;
fsView = hoodieTableFileSystemViewCache.computeIfAbsent(baseDir.toString(), key ->
FileSystemViewManager.createInMemoryFileSystemView(
engineContext,
finalMetaClient,
HoodieInputFormatUtils.buildMetadataConfig(getConf())
)
);
fsView = FileSystemViewManager.createInMemoryFileSystemView(engineContext,
metaClient, HoodieInputFormatUtils.buildMetadataConfig(getConf()));
String partition = FSUtils.getRelativePartitionPath(new Path(metaClient.getBasePath()), folder);
List<HoodieBaseFile> latestFiles = fsView.getLatestBaseFiles(partition).collect(Collectors.toList());
// populate the cache
@@ -215,6 +202,10 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
}
nonHoodiePathCache.add(folder.toString());
return true;
} finally {
if (fsView != null) {
fsView.close();
}
}
} else {
// files is at < 3 level depth in FS tree, can't be hoodie dataset