lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4448] Remove the latest commit refresh for timeline server (#6179)

Browse Source
This commit is contained in:
Danny Chan
2022-07-24 07:10:53 +08:00
committed by GitHub
parent 2d745057ea
commit a0ffd05b77
6 changed files with 4 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java
View File

@@ -121,7 +121,6 @@ public class RequestHandler {
String lastKnownInstantFromClient =
ctx.queryParam(RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, HoodieTimeline.INVALID_INSTANT_TS);
String timelineHashFromClient = ctx.queryParam(RemoteHoodieTableFileSystemView.TIMELINE_HASH, "");
String numInstantsFromClient = ctx.queryParam(RemoteHoodieTableFileSystemView.NUM_INSTANTS, "-1");
HoodieTimeline localTimeline =
viewManager.getFileSystemView(basePath).getTimeline().filterCompletedAndCompactionInstants();
if (LOG.isDebugEnabled()) {
@@ -135,10 +134,8 @@ public class RequestHandler {
}
String localTimelineHash = localTimeline.getTimelineHash();
// refresh if timeline hash mismatches and if local's last known instant < client's last known instant (if config is enabled)
if (!localTimelineHash.equals(timelineHashFromClient)
&& (!timelineServiceConfig.refreshTimelineBasedOnLatestCommit
|| localTimelineBehind(localTimeline, lastKnownInstantFromClient, numInstantsFromClient))) {
// refresh if timeline hash mismatches
if (!localTimelineHash.equals(timelineHashFromClient)) {
return true;
}
@@ -146,22 +143,6 @@ public class RequestHandler {
return !localTimeline.containsOrBeforeTimelineStarts(lastKnownInstantFromClient);
}
private static boolean localTimelineBehind(HoodieTimeline localTimeline, String lastKnownInstantFromClient, String numInstantsFromClient) {
String localLastKnownInstant = localTimeline.lastInstant().isPresent() ? localTimeline.lastInstant().get().getTimestamp()
: HoodieTimeline.INVALID_INSTANT_TS;
// Why comparing the num commits ?
// Assumes there are 4 commits on the timeline:
// timestamp(action): ts_0(commit), ts_1(commit), ts_2(clean), ts_3(commit)
// when ts_1 is in INFLIGHT state, ts_2 clean action is already finished,
// after ts_1 triggers #sync, the local timeline is refreshed as [ts_0, ts_2],
// when ts_1 switches state from INFLIGHT to COMPLETED, no #sync triggers.
// at ts_3, when the fs view snapshot is requested, the ts_3 client timeline should be [ts_0, ts_1, ts_2],
// if we only compare the latest commit, the local timeline is NOT behind, but the fs view is not complete
// because ts_1 is lost.
return HoodieTimeline.compareTimestamps(localLastKnownInstant, HoodieTimeline.LESSER_THAN, lastKnownInstantFromClient)
|| localTimeline.countInstants() < Integer.parseInt(numInstantsFromClient);
}
/**
* Syncs data-set view if local view is behind.
*/

10
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
View File

@@ -123,9 +123,6 @@ public class TimelineService {
@Parameter(names = {"--marker-parallelism", "-mdp"}, description = "Parallelism to use for reading and deleting marker files")
public int markerParallelism = 100;
@Parameter(names = {"--refreshTimelineBasedOnLatestCommit"}, description = "Refresh local timeline based on latest commit in addition to timeline hash value")
public boolean refreshTimelineBasedOnLatestCommit = true;
@Parameter(names = {"--help", "-h"})
public Boolean help = false;
@@ -150,7 +147,6 @@ public class TimelineService {
private int markerBatchNumThreads = 20;
private long markerBatchIntervalMs = 50L;
private int markerParallelism = 100;
private boolean refreshTimelineBasedOnLatestCommit = true;
public Builder() {
}
@@ -200,11 +196,6 @@ public class TimelineService {
return this;
}
public Builder refreshTimelineBasedOnLatestCommit(boolean refreshTimelineBasedOnLatestCommit) {
this.refreshTimelineBasedOnLatestCommit = refreshTimelineBasedOnLatestCommit;
return this;
}
public Builder enableMarkerRequests(boolean enableMarkerRequests) {
this.enableMarkerRequests = enableMarkerRequests;
return this;
@@ -240,7 +231,6 @@ public class TimelineService {
config.markerBatchNumThreads = this.markerBatchNumThreads;
config.markerBatchIntervalMs = this.markerBatchIntervalMs;
config.markerParallelism = this.markerParallelism;
config.refreshTimelineBasedOnLatestCommit = this.refreshTimelineBasedOnLatestCommit;
return config;
}
}