lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1276] [HUDI-1459] Make Clustering/ReplaceCommit and Metadata table be compatible (#2422)

Browse Source 
* [HUDI-1276] [HUDI-1459] Make Clustering/ReplaceCommit and Metadata table be compatible

* Use filesystemview and json format from metadata. Add tests

Co-authored-by: Satish Kotha <satishkotha@uber.com>
This commit is contained in:
vinoth chandar
2021-01-09 16:53:34 -08:00
committed by GitHub
parent 79ec7b4894
commit 65866c45ec
17 changed files with 301 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java
View File

@@ -299,6 +299,21 @@ public class FileSystemViewHandler {
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_BEFORE, new ViewHandler(ctx -> {
List<FileGroupDTO> dtos = sliceHandler.getReplacedFileGroupsBefore(
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(),
ctx.queryParam(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM,""),
ctx.queryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM,""));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_PARTITION, new ViewHandler(ctx -> {
List<FileGroupDTO> dtos = sliceHandler.getAllReplacedFileGroups(
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(),
ctx.queryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM,""));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.PENDING_CLUSTERING_FILEGROUPS, new ViewHandler(ctx -> {
List<ClusteringOpDTO> dtos = sliceHandler.getFileGroupsInPendingClustering(
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow());

10
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java
View File

@@ -94,6 +94,16 @@ public class FileSliceHandler extends Handler {
.collect(Collectors.toList());
}
public List<FileGroupDTO> getReplacedFileGroupsBefore(String basePath, String maxCommitTime, String partitionPath) {
return viewManager.getFileSystemView(basePath).getReplacedFileGroupsBefore(maxCommitTime, partitionPath).map(FileGroupDTO::fromFileGroup)
.collect(Collectors.toList());
}
public List<FileGroupDTO> getAllReplacedFileGroups(String basePath, String partitionPath) {
return viewManager.getFileSystemView(basePath).getAllReplacedFileGroups(partitionPath).map(FileGroupDTO::fromFileGroup)
.collect(Collectors.toList());
}
public List<ClusteringOpDTO> getFileGroupsInPendingClustering(String basePath) {
return viewManager.getFileSystemView(basePath).getFileGroupsInPendingClustering()
.map(fgInstant -> ClusteringOpDTO.fromClusteringOp(fgInstant.getLeft(), fgInstant.getRight()))