lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-690] Filter out inflight compaction in exporter (#1667)

Browse Source
This commit is contained in:
Raymond Xu
2020-05-26 09:23:34 -07:00
committed by GitHub
parent 46572d04c4
commit 6c450957ce
3 changed files with 7 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
View File

@@ -152,8 +152,8 @@ public class HoodieDefaultTimeline implements HoodieTimeline {
} }
/** /**
* Get all instants (commits, delta commits, clean, savepoint, rollback) that result in actions, in the active * Get all instants (commits, delta commits, compaction, clean, savepoint, rollback) that result in actions,
* timeline. * in the active timeline.
*/ */
public HoodieTimeline getAllCommitsTimeline() { public HoodieTimeline getAllCommitsTimeline() {
return getTimelineOfActions(CollectionUtils.createSet(COMMIT_ACTION, DELTA_COMMIT_ACTION, return getTimelineOfActions(CollectionUtils.createSet(COMMIT_ACTION, DELTA_COMMIT_ACTION,

4
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
View File

@@ -74,10 +74,10 @@ public class HoodieSnapshotCopier implements Serializable {
final SerializableConfiguration serConf = new SerializableConfiguration(jsc.hadoopConfiguration()); final SerializableConfiguration serConf = new SerializableConfiguration(jsc.hadoopConfiguration());
final HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), baseDir); final HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), baseDir);
final BaseFileOnlyView fsView = new HoodieTableFileSystemView(tableMetadata, final BaseFileOnlyView fsView = new HoodieTableFileSystemView(tableMetadata,
tableMetadata.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()); tableMetadata.getActiveTimeline().getCommitsAndCompactionTimeline().filterCompletedInstants());
// Get the latest commit // Get the latest commit
Option<HoodieInstant> latestCommit = Option<HoodieInstant> latestCommit =
tableMetadata.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().lastInstant(); tableMetadata.getActiveTimeline().getCommitsAndCompactionTimeline().filterCompletedInstants().lastInstant();
if (!latestCommit.isPresent()) { if (!latestCommit.isPresent()) {
LOG.warn("No commits present. Nothing to snapshot"); LOG.warn("No commits present. Nothing to snapshot");
return; return;

5
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
View File

@@ -146,7 +146,8 @@ public class HoodieSnapshotExporter {
private Option<String> getLatestCommitTimestamp(FileSystem fs, Config cfg) { private Option<String> getLatestCommitTimestamp(FileSystem fs, Config cfg) {
final HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), cfg.sourceBasePath); final HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), cfg.sourceBasePath);
Option<HoodieInstant> latestCommit = tableMetadata.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().lastInstant(); Option<HoodieInstant> latestCommit = tableMetadata.getActiveTimeline().getCommitsAndCompactionTimeline()
.filterCompletedInstants().lastInstant();
return latestCommit.isPresent() ? Option.of(latestCommit.get().getTimestamp()) : Option.empty(); return latestCommit.isPresent() ? Option.of(latestCommit.get().getTimestamp()) : Option.empty();
} }
@@ -251,7 +252,7 @@ public class HoodieSnapshotExporter {
FileSystem fs = FSUtils.getFs(cfg.sourceBasePath, jsc.hadoopConfiguration()); FileSystem fs = FSUtils.getFs(cfg.sourceBasePath, jsc.hadoopConfiguration());
HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), cfg.sourceBasePath); HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), cfg.sourceBasePath);
return new HoodieTableFileSystemView(tableMetadata, tableMetadata return new HoodieTableFileSystemView(tableMetadata, tableMetadata
.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()); .getActiveTimeline().getCommitsAndCompactionTimeline().filterCompletedInstants());
} }
public static void main(String[] args) throws IOException { public static void main(String[] args) throws IOException {