lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2252] Default consumes from the latest instant for flink streaming reader (#3368)

Browse Source
This commit is contained in:
swuferhong
2021-07-30 14:25:05 +08:00
committed by GitHub
parent 7bdae69053
commit 8b19ec9ca0
4 changed files with 83 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-flink/src/main/java/org/apache/hudi/source/StreamReadMonitoringFunction.java
View File

@@ -203,9 +203,10 @@ public class StreamReadMonitoringFunction
instantRange = InstantRange.getInstance(specifiedStart, instantToIssue.getTimestamp(),
InstantRange.RangeType.CLOSE_CLOSE);
} else {
// first time consume and no start commit,
// would consume all the snapshot data PLUS incremental data set
instantRange = null;
// first time consume and no start commit, consumes the latest incremental data set.
HoodieInstant latestCommitInstant = metaClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get();
instantRange = InstantRange.getInstance(latestCommitInstant.getTimestamp(), instantToIssue.getTimestamp(),
InstantRange.RangeType.CLOSE_CLOSE);
}
} else {
LOG.info("No new instant found for the table under path " + path + ", skip reading");