lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3366] Remove hardcoded logic of disabling metadata table in tests (#4792)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-02-15 13:41:47 -08:00
committed by GitHub
parent 538ec44fa8
commit 9a05940a74
25 changed files with 244 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
View File

@@ -19,8 +19,6 @@
package org.apache.hudi.client;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hudi.avro.model.HoodieArchivedMetaEntry;
import org.apache.hudi.avro.model.HoodieMergeArchiveFilePlan;
import org.apache.hudi.client.utils.MetadataConversionUtils;
@@ -59,6 +57,8 @@ import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -429,7 +429,7 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
.collect(Collectors.groupingBy(i -> Pair.of(i.getTimestamp(),
HoodieInstant.getComparableAction(i.getAction()))));
// If metadata table is enabled, do not archive instants which are more recent that the last compaction on the
// If metadata table is enabled, do not archive instants which are more recent than the last compaction on the
// metadata table.
if (config.isMetadataTableEnabled()) {
try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), config.getMetadataConfig(),
@@ -447,7 +447,7 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
throw new HoodieException("Error limiting instant archival based on metadata table", e);
}
}
return instants.flatMap(hoodieInstant ->
groupByTsAction.get(Pair.of(hoodieInstant.getTimestamp(),
HoodieInstant.getComparableAction(hoodieInstant.getAction()))).stream());