lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

HUDI-148 Small File selection logic for MOR must skip fileIds selected for pending compaction correctly

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-06-08 12:40:08 -07:00
committed by vinoth chandar
parent 8c9980f4f5
commit a1483f2c5f
11 changed files with 48 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/FileSystemViewHandler.java
View File

@@ -265,7 +265,8 @@ public class FileSystemViewHandler {
List<FileSliceDTO> dtos =
sliceHandler.getLatestFileSlicesBeforeOrOn(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(),
ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow(),
ctx.validatedQueryParam(MAX_INSTANT_PARAM).getOrThrow());
ctx.validatedQueryParam(MAX_INSTANT_PARAM).getOrThrow(),
Boolean.valueOf(ctx.validatedQueryParam(INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM).getOrThrow()));
writeValueAsString(ctx, dtos);
}, true));

6
hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/FileSliceHandler.java
View File

@@ -56,9 +56,9 @@ public class FileSliceHandler extends Handler {
}
public List<FileSliceDTO> getLatestFileSlicesBeforeOrOn(String basePath, String partitionPath,
String maxInstantTime) {
return viewManager.getFileSystemView(basePath).getLatestFileSlicesBeforeOrOn(partitionPath, maxInstantTime)
.map(FileSliceDTO::fromFileSlice).collect(Collectors.toList());
String maxInstantTime, boolean includeFileSlicesInPendingCompaction) {
return viewManager.getFileSystemView(basePath).getLatestFileSlicesBeforeOrOn(partitionPath, maxInstantTime,
includeFileSlicesInPendingCompaction).map(FileSliceDTO::fromFileSlice).collect(Collectors.toList());
}
public List<FileSliceDTO> getLatestUnCompactedFileSlices(String basePath, String partitionPath) {