lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1646] Provide mechanism to read uncommitted data through InputFormat (#2611)

Browse Source
This commit is contained in:
satishkotha
2021-03-04 17:43:31 -08:00
committed by GitHub
parent 89003bc780
commit 7cc75e0be2
7 changed files with 109 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
View File

@@ -157,6 +157,13 @@ public class HoodieDefaultTimeline implements HoodieTimeline {
details);
}
@Override
public HoodieDefaultTimeline findInstantsBeforeOrEquals(String instantTime) {
return new HoodieDefaultTimeline(instants.stream()
.filter(s -> HoodieTimeline.compareTimestamps(s.getTimestamp(), LESSER_THAN_OR_EQUALS, instantTime)),
details);
}
@Override
public HoodieTimeline filter(Predicate<HoodieInstant> filter) {
return new HoodieDefaultTimeline(instants.stream().filter(filter), details);

5
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
View File

@@ -176,6 +176,11 @@ public interface HoodieTimeline extends Serializable {
*/
HoodieTimeline findInstantsBefore(String instantTime);
/**
* Create new timeline with all instants before or equals specified time.
*/
HoodieTimeline findInstantsBeforeOrEquals(String instantTime);
/**
* Custom Filter of Instants.
*/

17
hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
View File

@@ -169,14 +169,21 @@ public class FileSystemViewManager {
public static HoodieTableFileSystemView createInMemoryFileSystemView(HoodieEngineContext engineContext, HoodieTableMetaClient metaClient,
HoodieMetadataConfig metadataConfig) {
return createInMemoryFileSystemViewWithTimeline(engineContext, metaClient, metadataConfig,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants());
}
public static HoodieTableFileSystemView createInMemoryFileSystemViewWithTimeline(HoodieEngineContext engineContext,
HoodieTableMetaClient metaClient,
HoodieMetadataConfig metadataConfig,
HoodieTimeline timeline) {
LOG.info("Creating InMemory based view for basePath " + metaClient.getBasePath());
if (metadataConfig.useFileListingMetadata()) {
return new HoodieMetadataFileSystemView(engineContext, metaClient,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
metadataConfig);
return new HoodieMetadataFileSystemView(engineContext, metaClient, timeline, metadataConfig);
}
return new HoodieTableFileSystemView(metaClient,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants());
return new HoodieTableFileSystemView(metaClient, timeline);
}
/**