lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-65] commitTime rename to instantTime (#1431)

Browse Source
This commit is contained in:
Zhiyuan Zhao
2020-03-23 09:06:00 +08:00
committed by GitHub
parent 38c3ccc51a
commit 0241b21f77
68 changed files with 673 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
View File

@@ -124,16 +124,16 @@ public class HoodieTestUtils {
return new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
}
public static void createCommitFiles(String basePath, String... commitTimes) throws IOException {
for (String commitTime : commitTimes) {
public static void createCommitFiles(String basePath, String... instantTimes) throws IOException {
for (String instantTime : instantTimes) {
new File(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ HoodieTimeline.makeRequestedCommitFileName(commitTime)).createNewFile();
+ HoodieTimeline.makeRequestedCommitFileName(instantTime)).createNewFile();
new File(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ HoodieTimeline.makeInflightCommitFileName(commitTime)).createNewFile();
+ HoodieTimeline.makeInflightCommitFileName(instantTime)).createNewFile();
new File(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCommitFileName(commitTime))
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCommitFileName(instantTime))
.createNewFile();
}
}
@@ -142,20 +142,20 @@ public class HoodieTestUtils {
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME).mkdirs();
}
public static void createInflightCommitFiles(String basePath, String... commitTimes) throws IOException {
public static void createInflightCommitFiles(String basePath, String... instantTimes) throws IOException {
for (String commitTime : commitTimes) {
for (String instantTime : instantTimes) {
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ HoodieTimeline.makeRequestedCommitFileName(commitTime)).createNewFile();
+ HoodieTimeline.makeRequestedCommitFileName(instantTime)).createNewFile();
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeInflightCommitFileName(
commitTime)).createNewFile();
instantTime)).createNewFile();
}
}
public static void createPendingCleanFiles(HoodieTableMetaClient metaClient, String... commitTimes) {
for (String commitTime : commitTimes) {
Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(commitTime),
HoodieTimeline.makeInflightCleanerFileName(commitTime)).forEach(f -> {
public static void createPendingCleanFiles(HoodieTableMetaClient metaClient, String... instantTimes) {
for (String instantTime : instantTimes) {
Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(instantTime),
HoodieTimeline.makeInflightCleanerFileName(instantTime)).forEach(f -> {
FSDataOutputStream os = null;
try {
Path commitFile = new Path(
@@ -179,59 +179,59 @@ public class HoodieTestUtils {
}
}
public static String createNewDataFile(String basePath, String partitionPath, String commitTime)
public static String createNewDataFile(String basePath, String partitionPath, String instantTime)
throws IOException {
String fileID = UUID.randomUUID().toString();
return createDataFile(basePath, partitionPath, commitTime, fileID);
return createDataFile(basePath, partitionPath, instantTime, fileID);
}
public static String createNewMarkerFile(String basePath, String partitionPath, String commitTime)
public static String createNewMarkerFile(String basePath, String partitionPath, String instantTime)
throws IOException {
String fileID = UUID.randomUUID().toString();
return createMarkerFile(basePath, partitionPath, commitTime, fileID);
return createMarkerFile(basePath, partitionPath, instantTime, fileID);
}
public static String createDataFile(String basePath, String partitionPath, String commitTime, String fileID)
public static String createDataFile(String basePath, String partitionPath, String instantTime, String fileID)
throws IOException {
String folderPath = basePath + "/" + partitionPath + "/";
new File(folderPath).mkdirs();
new File(folderPath + FSUtils.makeDataFileName(commitTime, DEFAULT_WRITE_TOKEN, fileID)).createNewFile();
new File(folderPath + FSUtils.makeDataFileName(instantTime, DEFAULT_WRITE_TOKEN, fileID)).createNewFile();
return fileID;
}
public static String createMarkerFile(String basePath, String partitionPath, String commitTime, String fileID)
public static String createMarkerFile(String basePath, String partitionPath, String instantTime, String fileID)
throws IOException {
String folderPath =
basePath + "/" + HoodieTableMetaClient.TEMPFOLDER_NAME + "/" + commitTime + "/" + partitionPath + "/";
basePath + "/" + HoodieTableMetaClient.TEMPFOLDER_NAME + "/" + instantTime + "/" + partitionPath + "/";
new File(folderPath).mkdirs();
File f = new File(folderPath + FSUtils.makeMarkerFile(commitTime, DEFAULT_WRITE_TOKEN, fileID));
File f = new File(folderPath + FSUtils.makeMarkerFile(instantTime, DEFAULT_WRITE_TOKEN, fileID));
f.createNewFile();
return f.getAbsolutePath();
}
public static String createNewLogFile(FileSystem fs, String basePath, String partitionPath, String commitTime,
public static String createNewLogFile(FileSystem fs, String basePath, String partitionPath, String instantTime,
String fileID, Option<Integer> version) throws IOException {
String folderPath = basePath + "/" + partitionPath + "/";
boolean makeDir = fs.mkdirs(new Path(folderPath));
if (!makeDir) {
throw new IOException("cannot create directory for path " + folderPath);
}
boolean createFile = fs.createNewFile(new Path(folderPath + FSUtils.makeLogFileName(fileID, ".log", commitTime,
boolean createFile = fs.createNewFile(new Path(folderPath + FSUtils.makeLogFileName(fileID, ".log", instantTime,
version.orElse(DEFAULT_LOG_VERSION), HoodieLogFormat.UNKNOWN_WRITE_TOKEN)));
if (!createFile) {
throw new IOException(
StringUtils.format("cannot create data file for commit %s and fileId %s", commitTime, fileID));
StringUtils.format("cannot create data file for commit %s and fileId %s", instantTime, fileID));
}
return fileID;
}
public static void createCompactionCommitFiles(FileSystem fs, String basePath, String... commitTimes)
public static void createCompactionCommitFiles(FileSystem fs, String basePath, String... instantTimes)
throws IOException {
for (String commitTime : commitTimes) {
for (String instantTime : instantTimes) {
boolean createFile = fs.createNewFile(new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ HoodieTimeline.makeCommitFileName(commitTime)));
+ HoodieTimeline.makeCommitFileName(instantTime)));
if (!createFile) {
throw new IOException("cannot create commit file for commit " + commitTime);
throw new IOException("cannot create commit file for commit " + instantTime);
}
}
}
@@ -244,67 +244,67 @@ public class HoodieTestUtils {
AvroUtils.serializeCompactionPlan(plan));
}
public static String getDataFilePath(String basePath, String partitionPath, String commitTime, String fileID) {
return basePath + "/" + partitionPath + "/" + FSUtils.makeDataFileName(commitTime, DEFAULT_WRITE_TOKEN, fileID);
public static String getDataFilePath(String basePath, String partitionPath, String instantTime, String fileID) {
return basePath + "/" + partitionPath + "/" + FSUtils.makeDataFileName(instantTime, DEFAULT_WRITE_TOKEN, fileID);
}
public static String getLogFilePath(String basePath, String partitionPath, String commitTime, String fileID,
public static String getLogFilePath(String basePath, String partitionPath, String instantTime, String fileID,
Option<Integer> version) {
return basePath + "/" + partitionPath + "/" + FSUtils.makeLogFileName(fileID, ".log", commitTime,
return basePath + "/" + partitionPath + "/" + FSUtils.makeLogFileName(fileID, ".log", instantTime,
version.orElse(DEFAULT_LOG_VERSION), HoodieLogFormat.UNKNOWN_WRITE_TOKEN);
}
public static String getCommitFilePath(String basePath, String commitTime) {
return basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + commitTime + HoodieTimeline.COMMIT_EXTENSION;
public static String getCommitFilePath(String basePath, String instantTime) {
return basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION;
}
public static String getInflightCommitFilePath(String basePath, String commitTime) {
return basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + commitTime
public static String getInflightCommitFilePath(String basePath, String instantTime) {
return basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime
+ HoodieTimeline.INFLIGHT_COMMIT_EXTENSION;
}
public static String getRequestedCompactionFilePath(String basePath, String commitTime) {
return basePath + "/" + HoodieTableMetaClient.AUXILIARYFOLDER_NAME + "/" + commitTime
public static String getRequestedCompactionFilePath(String basePath, String instantTime) {
return basePath + "/" + HoodieTableMetaClient.AUXILIARYFOLDER_NAME + "/" + instantTime
+ HoodieTimeline.INFLIGHT_COMMIT_EXTENSION;
}
public static boolean doesDataFileExist(String basePath, String partitionPath, String commitTime,
public static boolean doesDataFileExist(String basePath, String partitionPath, String instantTime,
String fileID) {
return new File(getDataFilePath(basePath, partitionPath, commitTime, fileID)).exists();
return new File(getDataFilePath(basePath, partitionPath, instantTime, fileID)).exists();
}
public static boolean doesLogFileExist(String basePath, String partitionPath, String commitTime, String fileID,
public static boolean doesLogFileExist(String basePath, String partitionPath, String instantTime, String fileID,
Option<Integer> version) {
return new File(getLogFilePath(basePath, partitionPath, commitTime, fileID, version)).exists();
return new File(getLogFilePath(basePath, partitionPath, instantTime, fileID, version)).exists();
}
public static boolean doesCommitExist(String basePath, String commitTime) {
public static boolean doesCommitExist(String basePath, String instantTime) {
return new File(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + commitTime + HoodieTimeline.COMMIT_EXTENSION)
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION)
.exists();
}
public static boolean doesInflightExist(String basePath, String commitTime) {
public static boolean doesInflightExist(String basePath, String instantTime) {
return new File(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + commitTime + HoodieTimeline.INFLIGHT_EXTENSION)
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.INFLIGHT_EXTENSION)
.exists();
}
public static void createCleanFiles(HoodieTableMetaClient metaClient, String basePath,
String commitTime, Configuration configuration)
String instantTime, Configuration configuration)
throws IOException {
createPendingCleanFiles(metaClient, commitTime);
createPendingCleanFiles(metaClient, instantTime);
Path commitFile = new Path(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCleanerFileName(commitTime));
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCleanerFileName(instantTime));
FileSystem fs = FSUtils.getFs(basePath, configuration);
try (FSDataOutputStream os = fs.create(commitFile, true)) {
HoodieCleanStat cleanStats = new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS,
DEFAULT_PARTITION_PATHS[rand.nextInt(DEFAULT_PARTITION_PATHS.length)], new ArrayList<>(), new ArrayList<>(),
new ArrayList<>(), commitTime);
new ArrayList<>(), instantTime);
// Create the clean metadata
HoodieCleanMetadata cleanMetadata =
CleanerUtils.convertCleanMetadata(metaClient, commitTime, Option.of(0L), Collections.singletonList(cleanStats));
CleanerUtils.convertCleanMetadata(metaClient, instantTime, Option.of(0L), Collections.singletonList(cleanStats));
// Write empty clean metadata
os.write(AvroUtils.serializeCleanMetadata(cleanMetadata).get());
}

4
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java
View File

@@ -37,7 +37,7 @@ public class TestHoodieWriteStat {
@Test
public void testSetPaths() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String basePathString = "/data/tables/some-hoodie-table";
String partitionPathString = "2017/12/31";
String fileName = UUID.randomUUID().toString();
@@ -46,7 +46,7 @@ public class TestHoodieWriteStat {
Path basePath = new Path(basePathString);
Path partitionPath = new Path(basePath, partitionPathString);
Path finalizeFilePath = new Path(partitionPath, FSUtils.makeDataFileName(commitTime, writeToken, fileName));
Path finalizeFilePath = new Path(partitionPath, FSUtils.makeDataFileName(instantTime, writeToken, fileName));
HoodieWriteStat writeStat = new HoodieWriteStat();
writeStat.setPath(basePath, finalizeFilePath);
assertEquals(finalizeFilePath, new Path(basePath, writeStat.getPath()));

4
hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
View File

@@ -407,7 +407,7 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
private List<HoodieInstant> getAllInstants() {
timeline = new HoodieActiveTimeline(metaClient);
List<HoodieInstant> allInstants = new ArrayList<>();
long commitTime = 1;
long instantTime = 1;
for (State state : State.values()) {
if (state == State.INVALID) {
continue;
@@ -432,7 +432,7 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
action = HoodieTimeline.COMMIT_ACTION;
}
allInstants.add(new HoodieInstant(state, action, String.format("%03d", commitTime++)));
allInstants.add(new HoodieInstant(state, action, String.format("%03d", instantTime++)));
}
}
return allInstants;

12
hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java
View File

@@ -100,12 +100,12 @@ public class SchemaTestUtil {
public static List<IndexedRecord> generateHoodieTestRecords(int from, int limit)
throws IOException, URISyntaxException {
List<IndexedRecord> records = generateTestRecords(from, limit);
String commitTime = HoodieActiveTimeline.createNewInstantTime();
String instantTime = HoodieActiveTimeline.createNewInstantTime();
Schema hoodieFieldsSchema = HoodieAvroUtils.addMetadataFields(getSimpleSchema());
return records.stream().map(s -> HoodieAvroUtils.rewriteRecord((GenericRecord) s, hoodieFieldsSchema)).map(p -> {
p.put(HoodieRecord.RECORD_KEY_METADATA_FIELD, UUID.randomUUID().toString());
p.put(HoodieRecord.PARTITION_PATH_METADATA_FIELD, "0000/00/00");
p.put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, commitTime);
p.put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, instantTime);
return p;
}).collect(Collectors.toList());
@@ -124,12 +124,12 @@ public class SchemaTestUtil {
}
public static List<IndexedRecord> updateHoodieTestRecords(List<String> oldRecordKeys, List<IndexedRecord> newRecords,
String commitTime) {
String instantTime) {
return newRecords.stream().map(p -> {
((GenericRecord) p).put(HoodieRecord.RECORD_KEY_METADATA_FIELD, oldRecordKeys.remove(0));
((GenericRecord) p).put(HoodieRecord.PARTITION_PATH_METADATA_FIELD, "0000/00/00");
((GenericRecord) p).put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, commitTime);
((GenericRecord) p).put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, instantTime);
return p;
}).collect(Collectors.toList());
@@ -173,9 +173,9 @@ public class SchemaTestUtil {
return new Schema.Parser().parse(SchemaTestUtil.class.getResourceAsStream("/timestamp-test-evolved.avsc"));
}
public static GenericRecord generateAvroRecordFromJson(Schema schema, int recordNumber, String commitTime,
public static GenericRecord generateAvroRecordFromJson(Schema schema, int recordNumber, String instantTime,
String fileId) throws IOException {
TestRecord record = new TestRecord(commitTime, recordNumber, fileId);
TestRecord record = new TestRecord(instantTime, recordNumber, fileId);
MercifulJsonConverter converter = new MercifulJsonConverter();
return converter.convert(record.toJsonString(), schema);
}

54
hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java
View File

@@ -68,16 +68,16 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test
public void testMakeDataFileName() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String fileName = UUID.randomUUID().toString();
assertEquals(FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName), fileName + "_" + TEST_WRITE_TOKEN + "_" + commitTime + ".parquet");
assertEquals(FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName), fileName + "_" + TEST_WRITE_TOKEN + "_" + instantTime + ".parquet");
}
@Test
public void testMaskFileName() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
int taskPartitionId = 2;
assertEquals(FSUtils.maskWithoutFileId(commitTime, taskPartitionId), "*_" + taskPartitionId + "_" + commitTime + ".parquet");
assertEquals(FSUtils.maskWithoutFileId(instantTime, taskPartitionId), "*_" + taskPartitionId + "_" + instantTime + ".parquet");
}
@Test
@@ -140,17 +140,17 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test
public void testGetCommitTime() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String fileName = UUID.randomUUID().toString();
String fullFileName = FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName);
assertEquals(FSUtils.getCommitTime(fullFileName), commitTime);
String fullFileName = FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName);
assertEquals(FSUtils.getCommitTime(fullFileName), instantTime);
}
@Test
public void testGetFileNameWithoutMeta() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String fileName = UUID.randomUUID().toString();
String fullFileName = FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName);
String fullFileName = FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName);
assertEquals(FSUtils.getFileId(fullFileName), fileName);
}
@@ -271,15 +271,15 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test
public void testDeleteOlderRollbackFiles() throws Exception {
String[] commitTimes = new String[]{"20160501010101", "20160501020101", "20160501030101", "20160501040101",
String[] instantTimes = new String[]{"20160501010101", "20160501020101", "20160501030101", "20160501040101",
"20160502020601", "20160502030601", "20160502040601", "20160502050601", "20160506030611",
"20160506040611", "20160506050611", "20160506060611"};
List<HoodieInstant> hoodieInstants = new ArrayList<>();
// create rollback files
for (String commitTime : commitTimes) {
for (String instantTime : instantTimes) {
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ commitTime + HoodieTimeline.ROLLBACK_EXTENSION).createNewFile();
hoodieInstants.add(new HoodieInstant(false, HoodieTimeline.ROLLBACK_ACTION, commitTime));
+ instantTime + HoodieTimeline.ROLLBACK_EXTENSION).createNewFile();
hoodieInstants.add(new HoodieInstant(false, HoodieTimeline.ROLLBACK_ACTION, instantTime));
}
FSUtils.deleteOlderRollbackMetaFiles(FSUtils.getFs(basePath, new Configuration()),
@@ -295,15 +295,15 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test
public void testDeleteOlderCleanMetaFiles() throws Exception {
String[] commitTimes = new String[]{"20160501010101", "20160501020101", "20160501030101", "20160501040101",
String[] instantTimes = new String[]{"20160501010101", "20160501020101", "20160501030101", "20160501040101",
"20160502020601", "20160502030601", "20160502040601", "20160502050601", "20160506030611",
"20160506040611", "20160506050611", "20160506060611"};
List<HoodieInstant> hoodieInstants = new ArrayList<>();
// create rollback files
for (String commitTime : commitTimes) {
for (String instantTime : instantTimes) {
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ commitTime + HoodieTimeline.CLEAN_EXTENSION).createNewFile();
hoodieInstants.add(new HoodieInstant(false, HoodieTimeline.CLEAN_ACTION, commitTime));
+ instantTime + HoodieTimeline.CLEAN_EXTENSION).createNewFile();
hoodieInstants.add(new HoodieInstant(false, HoodieTimeline.CLEAN_ACTION, instantTime));
}
FSUtils.deleteOlderCleanMetaFiles(FSUtils.getFs(basePath, new Configuration()),
basePath + "/.hoodie", hoodieInstants.stream());
@@ -318,7 +318,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test
public void testFileNameRelatedFunctions() throws Exception {
String commitTime = "20160501010101";
String instantTime = "20160501010101";
String partitionStr = "2016/05/01";
int taskPartitionId = 456;
String writeToken = "456";
@@ -328,13 +328,13 @@ public class TestFSUtils extends HoodieCommonTestHarness {
final String LOG_EXTENTION = "." + LOG_STR;
// data file name
String dataFileName = FSUtils.makeDataFileName(commitTime, writeToken, fileId);
assertTrue(commitTime.equals(FSUtils.getCommitTime(dataFileName)));
String dataFileName = FSUtils.makeDataFileName(instantTime, writeToken, fileId);
assertTrue(instantTime.equals(FSUtils.getCommitTime(dataFileName)));
assertTrue(fileId.equals(FSUtils.getFileId(dataFileName)));
String logFileName = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, commitTime, version, writeToken);
String logFileName = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, instantTime, version, writeToken);
assertTrue(FSUtils.isLogFile(new Path(logFileName)));
assertTrue(commitTime.equals(FSUtils.getBaseCommitTimeFromLogPath(new Path(logFileName))));
assertTrue(instantTime.equals(FSUtils.getBaseCommitTimeFromLogPath(new Path(logFileName))));
assertTrue(fileId.equals(FSUtils.getFileIdFromLogPath(new Path(logFileName))));
assertTrue(version == FSUtils.getFileVersionFromLog(new Path(logFileName)));
assertTrue(LOG_STR.equals(FSUtils.getFileExtensionFromLog(new Path(logFileName))));
@@ -342,16 +342,16 @@ public class TestFSUtils extends HoodieCommonTestHarness {
// create three versions of log file
String partitionPath = basePath + "/" + partitionStr;
new File(partitionPath).mkdirs();
String log1 = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, commitTime, 1, writeToken);
String log1 = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, instantTime, 1, writeToken);
new File(partitionPath + "/" + log1).createNewFile();
String log2 = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, commitTime, 2, writeToken);
String log2 = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, instantTime, 2, writeToken);
new File(partitionPath + "/" + log2).createNewFile();
String log3 = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, commitTime, 3, writeToken);
String log3 = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, instantTime, 3, writeToken);
new File(partitionPath + "/" + log3).createNewFile();
assertTrue(3 == FSUtils.getLatestLogVersion(FSUtils.getFs(basePath, new Configuration()),
new Path(partitionPath), fileId, LOG_EXTENTION, commitTime).get().getLeft());
new Path(partitionPath), fileId, LOG_EXTENTION, instantTime).get().getLeft());
assertTrue(4 == FSUtils.computeNextLogVersion(FSUtils.getFs(basePath, new Configuration()),
new Path(partitionPath), fileId, LOG_EXTENTION, commitTime));
new Path(partitionPath), fileId, LOG_EXTENTION, instantTime));
}
}

12
hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java
View File

@@ -74,15 +74,15 @@ public class TestRecord implements Serializable {
private TestNestedRecord testNestedRecord;
private String[] stringArray;
public TestRecord(String commitTime, int recordNumber, String fileId) {
this._hoodie_commit_time = commitTime;
public TestRecord(String instantTime, int recordNumber, String fileId) {
this._hoodie_commit_time = instantTime;
this._hoodie_record_key = "key" + recordNumber;
this._hoodie_partition_path = commitTime;
this._hoodie_partition_path = instantTime;
this._hoodie_file_name = fileId;
this._hoodie_commit_seqno = commitTime + recordNumber;
this._hoodie_commit_seqno = instantTime + recordNumber;
String commitTimeSuffix = "@" + commitTime;
int commitHashCode = commitTime.hashCode();
String commitTimeSuffix = "@" + instantTime;
int commitHashCode = instantTime.hashCode();
this.field1 = "field" + recordNumber;
this.field2 = "field" + recordNumber + commitTimeSuffix;

4
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
View File

@@ -215,7 +215,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
String newCommitTime = HoodieActiveTimeline.createNewInstantTime();
List<String> keysToBeUpdated = new ArrayList<>();
keysToBeUpdated.add(key);
// Update the commitTime for this record
// Update the instantTime for this record
List<IndexedRecord> updatedRecords =
SchemaTestUtil.updateHoodieTestRecords(keysToBeUpdated, recordsToUpdate, newCommitTime);
// Upsert this updated record
@@ -238,7 +238,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
// Upsert this updated record
SpillableMapTestUtils.upsertRecords(updatedRecords, records);
gRecord = (GenericRecord) records.get(key).getData().getInsertValue(schema).get();
// The record returned for this key should have the updated commitTime
// The record returned for this key should have the updated instantTime
assert newCommitTime.contentEquals(gRecord.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString());
}