lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-65] commitTime rename to instantTime (#1431)

Browse Source
This commit is contained in:
Zhiyuan Zhao
2020-03-23 09:06:00 +08:00
committed by GitHub
parent 38c3ccc51a
commit 0241b21f77
68 changed files with 673 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java
View File

@@ -160,15 +160,15 @@ public class HoodieFileGroup implements Serializable {
}
/**
* Obtain the latest file slice, upto a commitTime i.e <= maxCommitTime.
* Obtain the latest file slice, upto a instantTime i.e <= maxInstantTime.
*/
public Option<FileSlice> getLatestFileSliceBeforeOrOn(String maxCommitTime) {
public Option<FileSlice> getLatestFileSliceBeforeOrOn(String maxInstantTime) {
return Option.fromJavaOptional(getAllFileSlices().filter(slice -> HoodieTimeline
.compareTimestamps(slice.getBaseInstantTime(), maxCommitTime, HoodieTimeline.LESSER_OR_EQUAL)).findFirst());
.compareTimestamps(slice.getBaseInstantTime(), maxInstantTime, HoodieTimeline.LESSER_OR_EQUAL)).findFirst());
}
/**
* Obtain the latest file slice, upto a commitTime i.e < maxInstantTime.
* Obtain the latest file slice, upto an instantTime i.e < maxInstantTime.
*
* @param maxInstantTime Max Instant Time
* @return

4
hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
View File

@@ -65,9 +65,9 @@ public class HoodiePartitionMetadata {
/**
* Construct metadata object to be written out.
*/
public HoodiePartitionMetadata(FileSystem fs, String commitTime, Path basePath, Path partitionPath) {
public HoodiePartitionMetadata(FileSystem fs, String instantTime, Path basePath, Path partitionPath) {
this(fs, partitionPath);
props.setProperty(COMMIT_TIME_KEY, commitTime);
props.setProperty(COMMIT_TIME_KEY, instantTime);
props.setProperty(PARTITION_DEPTH_KEY, String.valueOf(partitionPath.depth() - basePath.depth()));
}

4
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java
View File

@@ -160,8 +160,8 @@ public class HoodieRecord<T extends HoodieRecordPayload> implements Serializable
return sb.toString();
}
public static String generateSequenceId(String commitTime, int partitionId, long recordIndex) {
return commitTime + "_" + partitionId + "_" + recordIndex;
public static String generateSequenceId(String instantTime, int partitionId, long recordIndex) {
return instantTime + "_" + partitionId + "_" + recordIndex;
}
public String getPartitionPath() {

42
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java
View File

@@ -142,7 +142,7 @@ public interface HoodieTimeline extends Serializable {
/**
* Create a new Timeline with all the instants after startTs.
*/
HoodieTimeline findInstantsAfter(String commitTime, int numCommits);
HoodieTimeline findInstantsAfter(String instantTime, int numCommits);
/**
* Custom Filter of Instants.
@@ -280,16 +280,16 @@ public interface HoodieTimeline extends Serializable {
return new HoodieInstant(true, instant.getAction(), instant.getTimestamp());
}
static String makeCommitFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.COMMIT_EXTENSION);
static String makeCommitFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.COMMIT_EXTENSION);
}
static String makeInflightCommitFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.INFLIGHT_COMMIT_EXTENSION);
static String makeInflightCommitFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.INFLIGHT_COMMIT_EXTENSION);
}
static String makeRequestedCommitFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.REQUESTED_COMMIT_EXTENSION);
static String makeRequestedCommitFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.REQUESTED_COMMIT_EXTENSION);
}
static String makeCleanerFileName(String instant) {
@@ -312,28 +312,28 @@ public interface HoodieTimeline extends Serializable {
return StringUtils.join(instant, HoodieTimeline.INFLIGHT_ROLLBACK_EXTENSION);
}
static String makeInflightSavePointFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.INFLIGHT_SAVEPOINT_EXTENSION);
static String makeInflightSavePointFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.INFLIGHT_SAVEPOINT_EXTENSION);
}
static String makeSavePointFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.SAVEPOINT_EXTENSION);
static String makeSavePointFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.SAVEPOINT_EXTENSION);
}
static String makeInflightDeltaFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.INFLIGHT_DELTA_COMMIT_EXTENSION);
static String makeInflightDeltaFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.INFLIGHT_DELTA_COMMIT_EXTENSION);
}
static String makeRequestedDeltaFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.REQUESTED_DELTA_COMMIT_EXTENSION);
static String makeRequestedDeltaFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.REQUESTED_DELTA_COMMIT_EXTENSION);
}
static String makeInflightCompactionFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.INFLIGHT_COMPACTION_EXTENSION);
static String makeInflightCompactionFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.INFLIGHT_COMPACTION_EXTENSION);
}
static String makeRequestedCompactionFileName(String commitTime) {
return StringUtils.join(commitTime, HoodieTimeline.REQUESTED_COMPACTION_EXTENSION);
static String makeRequestedCompactionFileName(String instantTime) {
return StringUtils.join(instantTime, HoodieTimeline.REQUESTED_COMPACTION_EXTENSION);
}
static String makeRestoreFileName(String instant) {
@@ -344,8 +344,8 @@ public interface HoodieTimeline extends Serializable {
return StringUtils.join(instant, HoodieTimeline.INFLIGHT_RESTORE_EXTENSION);
}
static String makeDeltaFileName(String commitTime) {
return commitTime + HoodieTimeline.DELTA_COMMIT_EXTENSION;
static String makeDeltaFileName(String instantTime) {
return instantTime + HoodieTimeline.DELTA_COMMIT_EXTENSION;
}
static String getCommitFromCommitFile(String commitFileName) {

10
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
View File

@@ -121,7 +121,7 @@ public interface HoodieLogFormat {
// File Id
private String logFileId;
// File Commit Time stamp
private String commitTime;
private String instantTime;
// version number for this log file. If not specified, then the current version will be
// computed by inspecting the file system
private Integer logVersion;
@@ -173,7 +173,7 @@ public interface HoodieLogFormat {
}
public WriterBuilder overBaseCommit(String baseCommit) {
this.commitTime = baseCommit;
this.instantTime = baseCommit;
return this;
}
@@ -195,7 +195,7 @@ public interface HoodieLogFormat {
if (logFileId == null) {
throw new IllegalArgumentException("FileID is not specified");
}
if (commitTime == null) {
if (instantTime == null) {
throw new IllegalArgumentException("BaseCommitTime is not specified");
}
if (fileExtension == null) {
@@ -212,7 +212,7 @@ public interface HoodieLogFormat {
if (logVersion == null) {
LOG.info("Computing the next log version for " + logFileId + " in " + parentPath);
Option<Pair<Integer, String>> versionAndWriteToken =
FSUtils.getLatestLogVersion(fs, parentPath, logFileId, fileExtension, commitTime);
FSUtils.getLatestLogVersion(fs, parentPath, logFileId, fileExtension, instantTime);
if (versionAndWriteToken.isPresent()) {
logVersion = versionAndWriteToken.get().getKey();
logWriteToken = versionAndWriteToken.get().getValue();
@@ -233,7 +233,7 @@ public interface HoodieLogFormat {
}
Path logPath = new Path(parentPath,
FSUtils.makeLogFileName(logFileId, fileExtension, commitTime, logVersion, logWriteToken));
FSUtils.makeLogFileName(logFileId, fileExtension, instantTime, logVersion, logWriteToken));
LOG.info("HoodieLogFile on path " + logPath);
HoodieLogFile logFile = new HoodieLogFile(logPath);

6
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
View File

@@ -123,14 +123,14 @@ public class HoodieArchivedTimeline extends HoodieDefaultTimeline {
}
private HoodieInstant readCommit(GenericRecord record, boolean loadDetails) {
final String commitTime = record.get(HoodiePartitionMetadata.COMMIT_TIME_KEY).toString();
final String instantTime = record.get(HoodiePartitionMetadata.COMMIT_TIME_KEY).toString();
final String action = record.get(ACTION_TYPE_KEY).toString();
if (loadDetails) {
Option.ofNullable(record.get(getMetadataKey(action))).map(actionData ->
this.readCommits.put(commitTime, actionData.toString().getBytes(StandardCharsets.UTF_8))
this.readCommits.put(instantTime, actionData.toString().getBytes(StandardCharsets.UTF_8))
);
}
return new HoodieInstant(false, action, commitTime);
return new HoodieInstant(false, action, instantTime);
}
private String getMetadataKey(String action) {

4
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
View File

@@ -130,9 +130,9 @@ public class HoodieDefaultTimeline implements HoodieTimeline {
}
@Override
public HoodieDefaultTimeline findInstantsAfter(String commitTime, int numCommits) {
public HoodieDefaultTimeline findInstantsAfter(String instantTime, int numCommits) {
return new HoodieDefaultTimeline(instants.stream()
.filter(s -> HoodieTimeline.compareTimestamps(s.getTimestamp(), commitTime, GREATER)).limit(numCommits),
.filter(s -> HoodieTimeline.compareTimestamps(s.getTimestamp(), instantTime, GREATER)).limit(numCommits),
details);
}

16
hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
View File

@@ -105,12 +105,12 @@ public class FSUtils {
return String.format("%d-%d-%d", taskPartitionId, stageId, taskAttemptId);
}
public static String makeDataFileName(String commitTime, String writeToken, String fileId) {
return String.format("%s_%s_%s.parquet", fileId, writeToken, commitTime);
public static String makeDataFileName(String instantTime, String writeToken, String fileId) {
return String.format("%s_%s_%s.parquet", fileId, writeToken, instantTime);
}
public static String makeMarkerFile(String commitTime, String writeToken, String fileId) {
return String.format("%s_%s_%s%s", fileId, writeToken, commitTime, HoodieTableMetaClient.MARKER_EXTN);
public static String makeMarkerFile(String instantTime, String writeToken, String fileId) {
return String.format("%s_%s_%s%s", fileId, writeToken, instantTime, HoodieTableMetaClient.MARKER_EXTN);
}
public static String translateMarkerToDataPath(String basePath, String markerPath, String instantTs) {
@@ -125,8 +125,8 @@ public class FSUtils {
HoodieFileFormat.PARQUET.getFileExtension());
}
public static String maskWithoutFileId(String commitTime, int taskPartitionId) {
return String.format("*_%s_%s%s", taskPartitionId, commitTime, HoodieFileFormat.PARQUET.getFileExtension());
public static String maskWithoutFileId(String instantTime, int taskPartitionId) {
return String.format("*_%s_%s%s", taskPartitionId, instantTime, HoodieFileFormat.PARQUET.getFileExtension());
}
public static String getCommitFromCommitFile(String commitFileName) {
@@ -282,7 +282,7 @@ public class FSUtils {
}
/**
* Get the first part of the file name in the log file. That will be the fileId. Log file do not have commitTime in
* Get the first part of the file name in the log file. That will be the fileId. Log file do not have instantTime in
* the file name.
*/
public static String getFileIdFromLogPath(Path path) {
@@ -304,7 +304,7 @@ public class FSUtils {
}
/**
* Get the first part of the file name in the log file. That will be the fileId. Log file do not have commitTime in
* Get the first part of the file name in the log file. That will be the fileId. Log file do not have instantTime in
* the file name.
*/
public static String getBaseCommitTimeFromLogPath(Path path) {

4
hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
View File

@@ -180,8 +180,8 @@ public class HoodieAvroUtils {
/**
* Adds the Hoodie commit metadata into the provided Generic Record.
*/
public static GenericRecord addCommitMetadataToRecord(GenericRecord record, String commitTime, String commitSeqno) {
record.put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, commitTime);
public static GenericRecord addCommitMetadataToRecord(GenericRecord record, String instantTime, String commitSeqno) {
record.put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, instantTime);
record.put(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, commitSeqno);
return record;
}