[HUDI-3183] Wrong result of HoodieArchivedTimeline loadInstants with TimeRangeFilter (#4521)
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
@@ -251,6 +251,53 @@ public class TestCommitsCommand extends CLIFunctionalTestHarness {
|
|||||||
assertEquals(expected, got);
|
assertEquals(expected, got);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testShowArchivedCommitsWithMultiCommitsFile() throws Exception {
|
||||||
|
// Generate archive
|
||||||
|
HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath1)
|
||||||
|
.withSchema(HoodieTestCommitMetadataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
|
||||||
|
.withCompactionConfig(HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 3).build())
|
||||||
|
.withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
|
||||||
|
.withRemoteServerPort(timelineServicePort).build())
|
||||||
|
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build())
|
||||||
|
.forTable("test-trip-table").build();
|
||||||
|
|
||||||
|
// generate data and metadata
|
||||||
|
Map<String, Integer[]> data = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
for (int i = 194; i >= 154; i--) {
|
||||||
|
data.put(String.valueOf(i), new Integer[] {i, i});
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
|
||||||
|
String key = entry.getKey();
|
||||||
|
Integer[] value = entry.getValue();
|
||||||
|
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath1, key, hadoopConf(),
|
||||||
|
Option.of(value[0]), Option.of(value[1]));
|
||||||
|
// archive
|
||||||
|
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
|
||||||
|
HoodieSparkTable table = HoodieSparkTable.create(cfg, context(), metaClient);
|
||||||
|
|
||||||
|
// need to create multi archive files
|
||||||
|
HoodieTimelineArchiveLog archiveLog = new HoodieTimelineArchiveLog(cfg, table);
|
||||||
|
archiveLog.archiveIfRequired(context());
|
||||||
|
}
|
||||||
|
|
||||||
|
CommandResult cr = shell().executeCommand(String.format("commits showarchived --startTs %s --endTs %s", "160", "174"));
|
||||||
|
assertTrue(cr.isSuccess());
|
||||||
|
assertEquals(3, metaClient.reloadActiveTimeline().getCommitsTimeline().countInstants(),
|
||||||
|
"There should 3 instants not be archived!");
|
||||||
|
|
||||||
|
Map<String, Integer[]> data2 = new LinkedHashMap<>();
|
||||||
|
for (int i = 174; i >= 161; i--) {
|
||||||
|
data2.put(String.valueOf(i), new Integer[] {i, i});
|
||||||
|
}
|
||||||
|
String expected = generateExpectData(1, data2);
|
||||||
|
expected = removeNonWordAndStripSpace(expected);
|
||||||
|
String got = removeNonWordAndStripSpace(cr.getResult().toString());
|
||||||
|
assertEquals(expected, got);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test case of 'commit showpartitions' command.
|
* Test case of 'commit showpartitions' command.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ import java.util.stream.Collectors;
|
|||||||
*/
|
*/
|
||||||
public class HoodieArchivedTimeline extends HoodieDefaultTimeline {
|
public class HoodieArchivedTimeline extends HoodieDefaultTimeline {
|
||||||
private static final Pattern ARCHIVE_FILE_PATTERN =
|
private static final Pattern ARCHIVE_FILE_PATTERN =
|
||||||
Pattern.compile("^\\.commits_\\.archive\\.([0-9]*)$");
|
Pattern.compile("^\\.commits_\\.archive\\.([0-9]+).*");
|
||||||
|
|
||||||
private static final String HOODIE_COMMIT_ARCHIVE_LOG_FILE_PREFIX = "commits";
|
private static final String HOODIE_COMMIT_ARCHIVE_LOG_FILE_PREFIX = "commits";
|
||||||
private static final String ACTION_TYPE_KEY = "actionType";
|
private static final String ACTION_TYPE_KEY = "actionType";
|
||||||
|
|||||||
Reference in New Issue
Block a user