1
0

[HUDI-4391] Incremental read from archived commits for flink (#6096)

This commit is contained in:
Danny Chan
2022-07-14 15:19:26 +08:00
committed by GitHub
parent ee956b8951
commit e70a427956
8 changed files with 461 additions and 40 deletions

View File

@@ -19,6 +19,7 @@
package org.apache.hudi.common.table.log;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.ValidationUtils;
import java.io.Serializable;
import java.util.Objects;
@@ -33,19 +34,15 @@ public abstract class InstantRange implements Serializable {
protected final String endInstant;
public InstantRange(String startInstant, String endInstant) {
this.startInstant = Objects.requireNonNull(startInstant);
this.endInstant = Objects.requireNonNull(endInstant);
this.startInstant = startInstant;
this.endInstant = endInstant;
}
public static InstantRange getInstance(String startInstant, String endInstant, RangeType rangeType) {
switch (rangeType) {
case OPEN_CLOSE:
return new OpenCloseRange(startInstant, endInstant);
case CLOSE_CLOSE:
return new CloseCloseRange(startInstant, endInstant);
default:
throw new AssertionError();
}
/**
* Returns the builder.
*/
public static Builder builder() {
return new Builder();
}
public String getStartInstant() {
@@ -65,14 +62,14 @@ public abstract class InstantRange implements Serializable {
/**
* Represents a range type.
*/
public enum RangeType {
public static enum RangeType {
OPEN_CLOSE, CLOSE_CLOSE
}
private static class OpenCloseRange extends InstantRange {
public OpenCloseRange(String startInstant, String endInstant) {
super(startInstant, endInstant);
super(Objects.requireNonNull(startInstant), endInstant);
}
@Override
@@ -84,10 +81,31 @@ public abstract class InstantRange implements Serializable {
}
}
private static class OpenCloseRangeNullableBoundary extends InstantRange {
public OpenCloseRangeNullableBoundary(String startInstant, String endInstant) {
super(startInstant, endInstant);
ValidationUtils.checkArgument(startInstant != null || endInstant != null,
"Start and end instants can not both be null");
}
@Override
public boolean isInRange(String instant) {
if (startInstant == null) {
return HoodieTimeline.compareTimestamps(instant, HoodieTimeline.LESSER_THAN_OR_EQUALS, endInstant);
} else if (endInstant == null) {
return HoodieTimeline.compareTimestamps(instant, HoodieTimeline.GREATER_THAN, startInstant);
} else {
return HoodieTimeline.compareTimestamps(instant, HoodieTimeline.GREATER_THAN, startInstant)
&& HoodieTimeline.compareTimestamps(instant, HoodieTimeline.LESSER_THAN_OR_EQUALS, endInstant);
}
}
}
private static class CloseCloseRange extends InstantRange {
public CloseCloseRange(String startInstant, String endInstant) {
super(startInstant, endInstant);
super(Objects.requireNonNull(startInstant), endInstant);
}
@Override
@@ -98,4 +116,78 @@ public abstract class InstantRange implements Serializable {
return HoodieTimeline.compareTimestamps(instant, HoodieTimeline.GREATER_THAN_OR_EQUALS, startInstant);
}
}
private static class CloseCloseRangeNullableBoundary extends InstantRange {
public CloseCloseRangeNullableBoundary(String startInstant, String endInstant) {
super(startInstant, endInstant);
ValidationUtils.checkArgument(startInstant != null || endInstant != null,
"Start and end instants can not both be null");
}
@Override
public boolean isInRange(String instant) {
if (startInstant == null) {
return HoodieTimeline.compareTimestamps(instant, HoodieTimeline.LESSER_THAN_OR_EQUALS, endInstant);
} else if (endInstant == null) {
return HoodieTimeline.compareTimestamps(instant, HoodieTimeline.GREATER_THAN_OR_EQUALS, startInstant);
} else {
return HoodieTimeline.compareTimestamps(instant, HoodieTimeline.GREATER_THAN_OR_EQUALS, startInstant)
&& HoodieTimeline.compareTimestamps(instant, HoodieTimeline.LESSER_THAN_OR_EQUALS, endInstant);
}
}
}
// -------------------------------------------------------------------------
// Inner Class
// -------------------------------------------------------------------------
/**
* Builder for {@link InstantRange}.
*/
public static class Builder {
private String startInstant;
private String endInstant;
private RangeType rangeType;
private boolean nullableBoundary = false;
private Builder() {
}
public Builder startInstant(String startInstant) {
this.startInstant = startInstant;
return this;
}
public Builder endInstant(String endInstant) {
this.endInstant = endInstant;
return this;
}
public Builder rangeType(RangeType rangeType) {
this.rangeType = rangeType;
return this;
}
public Builder nullableBoundary(boolean nullable) {
this.nullableBoundary = nullable;
return this;
}
public InstantRange build() {
ValidationUtils.checkState(this.rangeType != null, "Range type is required");
switch (rangeType) {
case OPEN_CLOSE:
return nullableBoundary
? new OpenCloseRangeNullableBoundary(startInstant, endInstant)
: new OpenCloseRange(startInstant, endInstant);
case CLOSE_CLOSE:
return nullableBoundary
? new CloseCloseRangeNullableBoundary(startInstant, endInstant)
: new CloseCloseRange(startInstant, endInstant);
default:
throw new AssertionError();
}
}
}
}