From bae0528013d125bbaad2b0fbba082fa03dbf93af Mon Sep 17 00:00:00 2001 From: Vinoth Chandar Date: Wed, 26 Apr 2017 11:00:50 -0700 Subject: [PATCH] Cleanup calls to HoodieTimeline.compareTimeStamps --- .../com/uber/hoodie/cli/commands/CommitsCommand.java | 4 ++-- .../com/uber/hoodie/cli/commands/HoodieSyncCommand.java | 2 +- .../src/main/java/com/uber/hoodie/HoodieWriteClient.java | 2 +- .../src/main/java/com/uber/hoodie/io/HoodieCleaner.java | 7 ++++--- hoodie-client/src/test/java/HoodieClientExample.java | 2 -- .../test/java/com/uber/hoodie/TestMergeOnReadTable.java | 2 +- .../java/com/uber/hoodie/io/TestHoodieCompactor.java | 4 ++-- .../java/com/uber/hoodie/table/TestCopyOnWriteTable.java | 5 ++--- .../com/uber/hoodie/avro/HoodieAvroWriteSupport.java | 1 - .../com/uber/hoodie/common/table/HoodieTimeline.java | 2 +- .../common/table/timeline/HoodieDefaultTimeline.java | 9 +++++---- .../common/table/view/HoodieTableFileSystemView.java | 4 ++-- .../hadoop/realtime/HoodieRealtimeRecordReader.java | 5 +---- .../com/uber/hoodie/utilities/HoodieSnapshotCopier.java | 3 +-- 14 files changed, 23 insertions(+), 29 deletions(-) diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java b/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java index 8a3d45887..195fb8a0c 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java +++ b/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java @@ -228,8 +228,8 @@ public class CommitsCommand implements CommandMarker { String sourceLatestCommit = sourceTimeline.getInstants().iterator().hasNext() ? "0" : sourceTimeline.lastInstant().get().getTimestamp(); - if (sourceLatestCommit != null && sourceTimeline - .compareTimestamps(targetLatestCommit, sourceLatestCommit, HoodieTimeline.GREATER)) { + if (sourceLatestCommit != null && + HoodieTimeline.compareTimestamps(targetLatestCommit, sourceLatestCommit, HoodieTimeline.GREATER)) { // source is behind the target List commitsToCatchup = targetTimeline.findInstantsAfter(sourceLatestCommit, Integer.MAX_VALUE) diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieSyncCommand.java b/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieSyncCommand.java index 6ce7d76e7..bbf18b8bc 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieSyncCommand.java +++ b/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieSyncCommand.java @@ -79,7 +79,7 @@ public class HoodieSyncCommand implements CommandMarker { String sourceLatestCommit = sourceTimeline.getInstants().iterator().hasNext() ? "0" : sourceTimeline.lastInstant().get().getTimestamp(); - if (sourceLatestCommit != null && sourceTimeline + if (sourceLatestCommit != null && HoodieTimeline .compareTimestamps(targetLatestCommit, sourceLatestCommit, HoodieTimeline.GREATER)) { // source is behind the target List commitsToCatchup = diff --git a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java b/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java index f734da622..d83d52eff 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java +++ b/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java @@ -470,7 +470,7 @@ public class HoodieWriteClient implements Seriali } // Cannot allow savepoint time on a commit that could have been cleaned - Preconditions.checkArgument(table.getActiveTimeline() + Preconditions.checkArgument(HoodieTimeline .compareTimestamps(commitTime, lastCommitRetained, HoodieTimeline.GREATER_OR_EQUAL), "Could not savepoint commit " + commitTime + " as this is beyond the lookup window " + lastCommitRetained); diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCleaner.java b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCleaner.java index db63cce69..c56f72b5f 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCleaner.java +++ b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCleaner.java @@ -187,8 +187,9 @@ public class HoodieCleaner> { } // Always keep the last commit - if (commitTimeline - .compareTimestamps(earliestCommitToRetain.getTimestamp(), fileCommitTime, + if (HoodieTimeline.compareTimestamps( + earliestCommitToRetain.getTimestamp(), + fileCommitTime, HoodieTimeline.GREATER)) { // this is a commit, that should be cleaned. deletePaths.add(String @@ -217,7 +218,7 @@ public class HoodieCleaner> { HoodieInstant commitTime) { for (HoodieDataFile file : fileList) { String fileCommitTime = FSUtils.getCommitTime(file.getFileName()); - if (commitTimeline.compareTimestamps(commitTime.getTimestamp(), fileCommitTime, + if (HoodieTimeline.compareTimestamps(commitTime.getTimestamp(), fileCommitTime, HoodieTimeline.GREATER)) { // fileList is sorted on the reverse, so the first commit we find <= commitTime is the one we want return fileCommitTime; diff --git a/hoodie-client/src/test/java/HoodieClientExample.java b/hoodie-client/src/test/java/HoodieClientExample.java index 1758a9b5e..32285e6e7 100644 --- a/hoodie-client/src/test/java/HoodieClientExample.java +++ b/hoodie-client/src/test/java/HoodieClientExample.java @@ -35,9 +35,7 @@ import org.apache.log4j.Logger; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; - import java.util.List; -import java.util.Properties; /** * Driver program that uses the Hoodie client with synthetic workload, and performs basic diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestMergeOnReadTable.java b/hoodie-client/src/test/java/com/uber/hoodie/TestMergeOnReadTable.java index a76dcf807..a63ea7a4a 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestMergeOnReadTable.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestMergeOnReadTable.java @@ -175,7 +175,7 @@ public class TestMergeOnReadTable { HoodieReadClient readClient = new HoodieReadClient(jsc, basePath, sqlContext); assertEquals("Expecting a single commit.", 1, readClient.listCommitsSince("000").size()); String latestCompactionCommitTime = readClient.latestCommit(); - assertTrue(metaClient.getActiveTimeline() + assertTrue(HoodieTimeline .compareTimestamps("000", latestCompactionCommitTime, HoodieTimeline.LESSER)); assertEquals("Must contain 200 records", 200, readClient.readSince("000").count()); } diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java index a230364f0..753b7c30c 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java @@ -183,8 +183,8 @@ public class TestHoodieCompactor { table = HoodieTable.getHoodieTable(metaClient, config); HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); - assertTrue("Compaction commit should be > than last insert", timeline - .compareTimestamps(timeline.lastInstant().get().getTimestamp(), newCommitTime, + assertTrue("Compaction commit should be > than last insert", + HoodieTimeline.compareTimestamps(timeline.lastInstant().get().getTimestamp(), newCommitTime, HoodieTimeline.GREATER)); for (String partitionPath : dataGen.getPartitionPaths()) { diff --git a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java b/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java index f8ab264e3..666f940a4 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java @@ -186,9 +186,8 @@ public class TestCopyOnWriteTable { for (File file : new File(basePath + "/2016/01/31").listFiles()) { if (file.getName().endsWith(".parquet")) { if (FSUtils.getFileId(file.getName()) - .equals(FSUtils.getFileId(parquetFile.getName())) && metadata - .getActiveTimeline().getCommitTimeline() - .compareTimestamps(FSUtils.getCommitTime(file.getName()), + .equals(FSUtils.getFileId(parquetFile.getName())) && + HoodieTimeline.compareTimestamps(FSUtils.getCommitTime(file.getName()), FSUtils.getCommitTime(parquetFile.getName()), HoodieTimeline.GREATER)) { updatedParquetFile = file; break; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/avro/HoodieAvroWriteSupport.java b/hoodie-common/src/main/java/com/uber/hoodie/avro/HoodieAvroWriteSupport.java index 5a5191655..6fb7a9b5f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/avro/HoodieAvroWriteSupport.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/avro/HoodieAvroWriteSupport.java @@ -23,7 +23,6 @@ import org.apache.parquet.avro.AvroWriteSupport; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.schema.MessageType; -import java.io.*; import java.util.HashMap; /** diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java index 1844ca8a1..93377c3eb 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java @@ -164,7 +164,7 @@ public interface HoodieTimeline extends Serializable { (commit1, commit2) -> commit1.compareTo(commit2) <= 0; BiPredicate LESSER = (commit1, commit2) -> commit1.compareTo(commit2) < 0; - default boolean compareTimestamps(String commit1, String commit2, + static boolean compareTimestamps(String commit1, String commit2, BiPredicate predicateToApply) { return predicateToApply.test(commit1, commit2); } diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieDefaultTimeline.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieDefaultTimeline.java index 0f43f9c4a..e250640c6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieDefaultTimeline.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieDefaultTimeline.java @@ -65,14 +65,15 @@ public class HoodieDefaultTimeline implements HoodieTimeline { @Override public HoodieDefaultTimeline findInstantsInRange(String startTs, String endTs) { return new HoodieDefaultTimeline(instants.stream().filter( - s -> compareTimestamps(s.getTimestamp(), startTs, GREATER) && compareTimestamps( + s -> HoodieTimeline.compareTimestamps(s.getTimestamp(), startTs, GREATER) && + HoodieTimeline.compareTimestamps( s.getTimestamp(), endTs, LESSER_OR_EQUAL)), details); } @Override public HoodieDefaultTimeline findInstantsAfter(String commitTime, int numCommits) { return new HoodieDefaultTimeline( - instants.stream().filter(s -> compareTimestamps(s.getTimestamp(), commitTime, GREATER)) + instants.stream().filter(s -> HoodieTimeline.compareTimestamps(s.getTimestamp(), commitTime, GREATER)) .limit(numCommits), details); } @@ -131,8 +132,8 @@ public class HoodieDefaultTimeline implements HoodieTimeline { @Override public boolean isBeforeTimelineStarts(String instant) { Optional firstCommit = firstInstant(); - return firstCommit.isPresent() && compareTimestamps(instant, - firstCommit.get().getTimestamp(), LESSER); + return firstCommit.isPresent() && + HoodieTimeline.compareTimestamps(instant, firstCommit.get().getTimestamp(), LESSER); } diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java index 4b10808b3..0316658a0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java @@ -165,7 +165,7 @@ public class HoodieTableFileSystemView implements TableFileSystemView, Serializa visibleActiveCommitTimeline.lastInstant().get().getTimestamp()) .map((Function, Optional>) fss -> { for (HoodieDataFile fs1 : fss) { - if (visibleActiveCommitTimeline + if (HoodieTimeline .compareTimestamps(fs1.getCommitTime(), maxCommitToReturn, HoodieTimeline.LESSER_OR_EQUAL)) { return Optional.of(fs1); @@ -248,7 +248,7 @@ public class HoodieTableFileSystemView implements TableFileSystemView, Serializa .flatMap(fileStatus -> { HoodieDataFile dataFile = new HoodieDataFile(fileStatus); if (visibleActiveCommitTimeline.containsOrBeforeTimelineStarts(dataFile.getCommitTime()) - && visibleActiveCommitTimeline + && HoodieTimeline .compareTimestamps(dataFile.getCommitTime(), maxCommitTime, HoodieTimeline.LESSER_OR_EQUAL)) { return Stream.of(Pair.of(dataFile.getFileId(), dataFile)); diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReader.java b/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReader.java index 3d619ec82..312720441 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReader.java +++ b/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReader.java @@ -106,9 +106,6 @@ public class HoodieRealtimeRecordReader implements RecordReader