diff --git a/hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java b/hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java index f44e06280..5f3a27eb9 100644 --- a/hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java @@ -18,19 +18,21 @@ package org.apache.hudi; -import java.io.IOException; -import java.io.Serializable; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.client.embedded.EmbeddedTimelineService; import org.apache.hudi.client.utils.ClientUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; + +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; +import java.io.IOException; +import java.io.Serializable; + /** * Abstract class taking care of holding common member variables (FileSystem, SparkContext, HoodieConfigs) Also, manages * embedded timeline-server if enabled. diff --git a/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java b/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java index 72af619d6..603440851 100644 --- a/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java @@ -18,19 +18,6 @@ package org.apache.hudi; -import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; - -import com.google.common.base.Preconditions; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.client.embedded.EmbeddedTimelineService; @@ -54,10 +41,25 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.func.OperationResult; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; + /** * Client to perform admin operations related to compaction */ diff --git a/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java index 00a04947f..6fae5ac7f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java @@ -18,11 +18,6 @@ package org.apache.hudi; -import com.codahale.metrics.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import java.io.IOException; -import java.util.List; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanerPlan; import org.apache.hudi.client.embedded.EmbeddedTimelineService; @@ -38,10 +33,17 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.metrics.HoodieMetrics; import org.apache.hudi.table.HoodieTable; + +import com.codahale.metrics.Timer; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; +import java.io.IOException; +import java.util.List; + public class HoodieCleanClient extends AbstractHoodieClient { private static Logger logger = LogManager.getLogger(HoodieCleanClient.class); diff --git a/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java index 819279b25..4bfdc3229 100644 --- a/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java @@ -18,10 +18,6 @@ package org.apache.hudi; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.client.embedded.EmbeddedTimelineService; import org.apache.hudi.common.model.HoodieDataFile; @@ -38,6 +34,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieIndexException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; @@ -48,6 +45,12 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.types.StructType; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java index 600c7d24e..3ccba7d0e 100644 --- a/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java @@ -18,21 +18,6 @@ package org.apache.hudi; -import com.codahale.metrics.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.text.ParseException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieRestoreMetadata; @@ -77,6 +62,11 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.UserDefinedBulkInsertPartitioner; import org.apache.hudi.table.WorkloadProfile; import org.apache.hudi.table.WorkloadStat; + +import com.codahale.metrics.Timer; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; @@ -85,6 +75,19 @@ import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.storage.StorageLevel; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/WriteStatus.java b/hudi-client/src/main/java/org/apache/hudi/WriteStatus.java index 039c4ec65..7efa31f92 100644 --- a/hudi-client/src/main/java/org/apache/hudi/WriteStatus.java +++ b/hudi-client/src/main/java/org/apache/hudi/WriteStatus.java @@ -18,16 +18,17 @@ package org.apache.hudi; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.util.Option; + import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; -import org.apache.hudi.common.model.HoodieKey; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieWriteStat; -import org.apache.hudi.common.util.Option; /** * Status of a write operation. diff --git a/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java index 9f7b03b37..54aef32db 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java @@ -18,18 +18,20 @@ package org.apache.hudi.client.embedded; -import java.io.IOException; -import org.apache.hadoop.conf.Configuration; import org.apache.hudi.common.SerializableConfiguration; import org.apache.hudi.common.table.view.FileSystemViewManager; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; import org.apache.hudi.common.util.NetworkUtils; import org.apache.hudi.timeline.service.TimelineService; + +import org.apache.hadoop.conf.Configuration; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; +import java.io.IOException; + /** * Timeline Service that runs as part of write client */ diff --git a/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java index 2c45b1362..fe2998e69 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java @@ -20,6 +20,7 @@ package org.apache.hudi.client.utils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.config.HoodieWriteConfig; + import org.apache.spark.api.java.JavaSparkContext; public class ClientUtils { diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java index 8ae7df6cb..2f1c3e7e9 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java @@ -18,17 +18,20 @@ package org.apache.hudi.config; -import com.google.common.base.Preconditions; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.Properties; -import javax.annotation.concurrent.Immutable; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.io.compact.strategy.CompactionStrategy; import org.apache.hudi.io.compact.strategy.LogFileSizeBasedCompactionStrategy; +import com.google.common.base.Preconditions; + +import javax.annotation.concurrent.Immutable; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Properties; + /** * Compaction related config */ diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java index 1b4306448..9fd32cb96 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java @@ -18,11 +18,12 @@ package org.apache.hudi.config; +import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; + import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; -import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; public class HoodieHBaseIndexConfig extends DefaultHoodieConfig { diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java index 24b98f6e5..de9895509 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java @@ -18,12 +18,15 @@ package org.apache.hudi.config; +import org.apache.hudi.index.HoodieIndex; + +import javax.annotation.concurrent.Immutable; + import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; -import javax.annotation.concurrent.Immutable; -import org.apache.hudi.index.HoodieIndex; + /** * Indexing related config diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java index be6b96edd..43b28cc37 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java @@ -18,13 +18,15 @@ package org.apache.hudi.config; +import org.apache.spark.SparkEnv; +import org.apache.spark.util.Utils; + +import javax.annotation.concurrent.Immutable; + import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; -import javax.annotation.concurrent.Immutable; -import org.apache.spark.SparkEnv; -import org.apache.spark.util.Utils; /** * Memory related config diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java index 903b923bd..8b911e8d1 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java @@ -18,12 +18,14 @@ package org.apache.hudi.config; +import org.apache.hudi.metrics.MetricsReporterType; + +import javax.annotation.concurrent.Immutable; + import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; -import javax.annotation.concurrent.Immutable; -import org.apache.hudi.metrics.MetricsReporterType; /** * Fetch the configurations used by the Metrics system. diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index a391bca2a..05c8406bc 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -18,14 +18,6 @@ package org.apache.hudi.config; -import com.google.common.base.Preconditions; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.util.Map; -import java.util.Properties; -import javax.annotation.concurrent.Immutable; import org.apache.hudi.HoodieWriteClient; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieCleaningPolicy; @@ -35,9 +27,20 @@ import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.io.compact.strategy.CompactionStrategy; import org.apache.hudi.metrics.MetricsReporterType; + +import com.google.common.base.Preconditions; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.spark.storage.StorageLevel; +import javax.annotation.concurrent.Immutable; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Properties; + /** * Class storing configs for the {@link HoodieWriteClient} */ diff --git a/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java b/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java index b1230f719..c40bb2a3a 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java @@ -18,15 +18,17 @@ package org.apache.hudi.func; -import java.util.Iterator; -import java.util.List; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.api.java.function.Function2; +import java.util.Iterator; +import java.util.List; + /** * Map function that handles a sorted stream of HoodieRecords */ diff --git a/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java b/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java index 13915a0ca..8e1c19103 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java @@ -18,12 +18,6 @@ package org.apache.hudi.func; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.function.Function; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; @@ -36,6 +30,14 @@ import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.io.HoodieWriteHandle; import org.apache.hudi.table.HoodieTable; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; + /** * Lazy Iterable, that writes a stream of HoodieRecords sorted by the partitionPath, into new files. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java b/hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java index 384502082..07c801f26 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java @@ -18,9 +18,6 @@ package org.apache.hudi.func; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; @@ -28,6 +25,10 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.HoodieAppendHandle; import org.apache.hudi.table.HoodieTable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + /** * Lazy Iterable, that writes a stream of HoodieRecords sorted by the partitionPath, into new log files. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java b/hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java index fba7f581b..ff1da05be 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java @@ -18,9 +18,10 @@ package org.apache.hudi.func; -import java.io.Serializable; import org.apache.hudi.common.util.Option; +import java.io.Serializable; + /** * Holds Operation result. Used as a result container for Compaction Admin Client (running as part of Spark-launcher * process) to communicate results back to Hoodie CLI process. diff --git a/hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java b/hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java index ce5067283..56552e3ea 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java @@ -18,12 +18,14 @@ package org.apache.hudi.func; -import java.io.IOException; -import java.util.Iterator; import org.apache.hudi.common.util.queue.BoundedInMemoryQueue; import org.apache.hudi.exception.HoodieIOException; + import org.apache.parquet.hadoop.ParquetReader; +import java.io.IOException; +import java.util.Iterator; + /** * This class wraps a parquet reader and provides an iterator based api to read from a parquet file. This is used in * {@link BoundedInMemoryQueue} diff --git a/hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java b/hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java index 2f57d17bb..9741f7ce9 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java @@ -18,17 +18,19 @@ package org.apache.hudi.func; -import java.util.Iterator; -import java.util.function.Function; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor; import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; import org.apache.hudi.common.util.queue.BoundedInMemoryQueueProducer; import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer; import org.apache.hudi.config.HoodieWriteConfig; + import org.apache.spark.TaskContext; import org.apache.spark.TaskContext$; +import java.util.Iterator; +import java.util.function.Function; + public class SparkBoundedInMemoryExecutor extends BoundedInMemoryExecutor { // Need to set current spark thread's TaskContext into newly launched thread so that new thread can access diff --git a/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java index 68612d772..235b3fb56 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java @@ -18,7 +18,6 @@ package org.apache.hudi.index; -import java.io.Serializable; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieKey; @@ -32,10 +31,13 @@ import org.apache.hudi.index.bloom.HoodieBloomIndex; import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex; import org.apache.hudi.index.hbase.HBaseIndex; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import java.io.Serializable; + /** * Base class for different types of indexes to determine the mapping from uuid */ diff --git a/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java index 91cd8a892..a8cd6e554 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java @@ -18,11 +18,6 @@ package org.apache.hudi.index; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -32,12 +27,19 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function; import org.apache.spark.api.java.function.Function2; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + /** * Hoodie Index implementation backed by an in-memory Hash map. *

diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java index 007c1af41..3a51cb9e3 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java @@ -19,6 +19,7 @@ package org.apache.hudi.index.bloom; import com.google.common.base.Objects; + import java.io.Serializable; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java index abceb1a2c..a4b80c649 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java @@ -18,18 +18,20 @@ package org.apache.hudi.index.bloom; +import org.apache.hudi.common.util.collection.Pair; + import com.google.common.annotations.VisibleForTesting; import com.google.common.hash.Hashing; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.spark.Partitioner; + import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hudi.common.util.collection.Pair; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -import org.apache.spark.Partitioner; /** * Partitions bloom filter checks by spreading out comparisons across buckets of work. diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java index ee3007d96..aec8cc36c 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java @@ -18,16 +18,6 @@ package org.apache.hudi.index.bloom; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.mapping; -import static java.util.stream.Collectors.toList; - -import com.google.common.annotations.VisibleForTesting; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -42,6 +32,8 @@ import org.apache.hudi.exception.MetadataNotFoundException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.io.HoodieRangeInfoHandle; import org.apache.hudi.table.HoodieTable; + +import com.google.common.annotations.VisibleForTesting; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; @@ -49,8 +41,19 @@ import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.storage.StorageLevel; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import scala.Tuple2; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; + /** * Indexing mechanism based on bloom filter. Each parquet file includes its row_key bloom filter in its metadata. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java index 6da33da6f..2a289fee2 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java @@ -18,9 +18,6 @@ package org.apache.hudi.index.bloom; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; @@ -30,7 +27,13 @@ import org.apache.hudi.func.LazyIterableIterator; import org.apache.hudi.io.HoodieKeyLookupHandle; import org.apache.hudi.io.HoodieKeyLookupHandle.KeyLookupResult; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.api.java.function.Function2; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java index a23143ba8..ec308c600 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java @@ -18,13 +18,6 @@ package org.apache.hudi.index.bloom; -import com.google.common.annotations.VisibleForTesting; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Collectors; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; @@ -35,9 +28,19 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.table.HoodieTable; + +import com.google.common.annotations.VisibleForTesting; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java b/hudi-client/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java index 4686c1fc6..8fdf25c0f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java @@ -19,6 +19,7 @@ package org.apache.hudi.index.hbase; import org.apache.hudi.config.HoodieWriteConfig; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java index c40d8356c..beb5b4a9c 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java @@ -18,25 +18,6 @@ package org.apache.hudi.index.hbase; -import com.google.common.annotations.VisibleForTesting; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -54,6 +35,20 @@ import org.apache.hudi.exception.HoodieDependentSystemUnavailableException; import org.apache.hudi.exception.HoodieIndexException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; @@ -61,6 +56,14 @@ import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function2; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index 7906aa2a5..62c9e6b38 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -18,16 +18,6 @@ package org.apache.hudi.io; -import com.google.common.collect.Maps; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieDeltaWriteStat; @@ -52,11 +42,23 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieAppendException; import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.table.HoodieTable; + +import com.google.common.collect.Maps; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; import org.apache.spark.util.SizeEstimator; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + /** * IO Operation to append data onto an existing file. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java index f82bb6e06..c79b39f26 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java @@ -18,13 +18,6 @@ package org.apache.hudi.io; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; @@ -45,9 +38,18 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.table.HoodieTable; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + /** * Cleaner is responsible for garbage collecting older files in a given partition path, such that *

diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java index 2d2f1d4a1..c69294926 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java @@ -18,20 +18,6 @@ package org.apache.hudi.io; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieRollbackMetadata; @@ -57,10 +43,26 @@ import org.apache.hudi.exception.HoodieCommitException; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.table.HoodieTable; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * Archiver to bound the growth of .commit files */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java index 9484ffa01..06601b182 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java @@ -18,11 +18,6 @@ package org.apache.hudi.io; -import java.io.IOException; -import java.util.Iterator; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodieRecord; @@ -37,10 +32,17 @@ import org.apache.hudi.exception.HoodieInsertException; import org.apache.hudi.io.storage.HoodieStorageWriter; import org.apache.hudi.io.storage.HoodieStorageWriterFactory; import org.apache.hudi.table.HoodieTable; + +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; +import java.io.IOException; +import java.util.Iterator; + public class HoodieCreateHandle extends HoodieWriteHandle { private static Logger logger = LogManager.getLogger(HoodieCreateHandle.class); diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java index 68a924589..cff91f3b4 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java @@ -18,11 +18,12 @@ package org.apache.hudi.io; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; +import org.apache.hadoop.fs.FileSystem; + public abstract class HoodieIOHandle { protected final String instantTime; diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java index 20ab3bc06..08e980251 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java @@ -18,12 +18,6 @@ package org.apache.hudi.io; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.BloomFilter; import org.apache.hudi.common.model.HoodieDataFile; import org.apache.hudi.common.model.HoodieRecordPayload; @@ -34,9 +28,17 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieIndexException; import org.apache.hudi.table.HoodieTable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + /** * Takes a bunch of keys and returns ones that are present in the file group. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java index 8821c4e0b..800c893e7 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java @@ -18,15 +18,6 @@ package org.apache.hudi.io; -import java.io.IOException; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieDataFile; import org.apache.hudi.common.model.HoodiePartitionMetadata; @@ -47,10 +38,21 @@ import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.io.storage.HoodieStorageWriter; import org.apache.hudi.io.storage.HoodieStorageWriterFactory; import org.apache.hudi.table.HoodieTable; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; +import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + @SuppressWarnings("Duplicates") public class HoodieMergeHandle extends HoodieWriteHandle { diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java index 8ae9f5b78..ea1d84ea8 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java @@ -18,7 +18,6 @@ package org.apache.hudi.io; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieDataFile; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.ParquetUtils; @@ -26,6 +25,8 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; +import org.apache.hadoop.fs.Path; + /** * Extract range information for a given file slice */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieReadHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieReadHandle.java index fc91e80bf..2bfc46d53 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieReadHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieReadHandle.java @@ -18,13 +18,14 @@ package org.apache.hudi.io; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.model.HoodieDataFile; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; +import org.apache.hadoop.fs.FileSystem; + /** * Base class for read operations done logically on the file group. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java index 02c629703..95b994aee 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java @@ -18,12 +18,6 @@ package org.apache.hudi.io; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; @@ -36,10 +30,18 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.table.HoodieTable; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; +import java.io.IOException; + /** * Base class for all write operations logically performed at the file group level. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java index a674c04b7..b62f8b487 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java @@ -18,17 +18,19 @@ package org.apache.hudi.io.compact; -import java.io.IOException; -import java.io.Serializable; -import java.util.Set; import org.apache.hudi.WriteStatus; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import java.io.IOException; +import java.io.Serializable; +import java.util.Set; + /** * A HoodieCompactor runs compaction on a hoodie table */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieRealtimeTableCompactor.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieRealtimeTableCompactor.java index dc68c5b98..2f8f8d092 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieRealtimeTableCompactor.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieRealtimeTableCompactor.java @@ -18,21 +18,6 @@ package org.apache.hudi.io.compact; -import static java.util.stream.Collectors.toList; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.WriteStatus; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; @@ -55,6 +40,13 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.compact.strategy.CompactionStrategy; import org.apache.hudi.table.HoodieCopyOnWriteTable; import org.apache.hudi.table.HoodieTable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -63,6 +55,16 @@ import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.util.AccumulatorV2; import org.apache.spark.util.LongAccumulator; +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static java.util.stream.Collectors.toList; + /** * HoodieRealtimeTableCompactor compacts a hoodie table with merge on read storage. Computes all possible compactions, * passes it through a CompactionFilter and executes all the compactions and writes a new version of base files and make diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java index 0e179773b..0d51df599 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java @@ -18,12 +18,14 @@ package org.apache.hudi.io.compact.strategy; -import com.google.common.collect.Lists; -import java.util.List; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.config.HoodieWriteConfig; +import com.google.common.collect.Lists; + +import java.util.List; + /** * CompactionStrategy which looks at total IO to be done for the compaction (read + write) and limits the list of * compactions to be under a configured limit on the IO diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java index 49b887ec9..abc7d9275 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java @@ -18,7 +18,12 @@ package org.apache.hudi.io.compact.strategy; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; + import com.google.common.annotations.VisibleForTesting; + import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Comparator; @@ -26,9 +31,6 @@ import java.util.Date; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.apache.hudi.avro.model.HoodieCompactionOperation; -import org.apache.hudi.avro.model.HoodieCompactionPlan; -import org.apache.hudi.config.HoodieWriteConfig; /** * This strategy ensures that the last N partitions are picked up even if there are later partitions created for the diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java index 9d7de66e0..bdd24970c 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java @@ -18,10 +18,6 @@ package org.apache.hudi.io.compact.strategy; -import com.google.common.collect.Maps; -import java.io.Serializable; -import java.util.List; -import java.util.Map; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.model.HoodieDataFile; @@ -32,6 +28,12 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.compact.HoodieRealtimeTableCompactor; +import com.google.common.collect.Maps; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + /** * Strategy for compaction. Pluggable implementation to define how compaction should be done. The over-ridden * implementations of this abstract class can capture the relevant metrics to order and filter the final list of diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java index a72bafe5e..3a1f425b2 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java @@ -18,7 +18,13 @@ package org.apache.hudi.io.compact.strategy; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; + import com.google.common.annotations.VisibleForTesting; + import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Comparator; @@ -27,10 +33,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; -import org.apache.hudi.avro.model.HoodieCompactionOperation; -import org.apache.hudi.avro.model.HoodieCompactionPlan; -import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.exception.HoodieException; /** * This strategy orders compactions in reverse order of creation of Hive Partitions. It helps to compact data in latest diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java index 42caf011a..b7b3863aa 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java @@ -18,10 +18,6 @@ package org.apache.hudi.io.compact.strategy; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.model.HoodieDataFile; @@ -29,6 +25,11 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + /** * LogFileSizeBasedCompactionStrategy orders the compactions based on the total log files size and limits the * compactions within a configured IO bound diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java index 73b13bde8..e2e61d5e0 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java @@ -18,11 +18,12 @@ package org.apache.hudi.io.compact.strategy; -import java.util.List; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.config.HoodieWriteConfig; +import java.util.List; + /** * UnBoundedCompactionStrategy will not change ordering or filter any compaction. It is a pass-through and will compact * all the base files which has a log file. This usually means no-intelligence on compaction. diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java index 1650b4f4e..ace772e09 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java @@ -18,13 +18,14 @@ package org.apache.hudi.io.compact.strategy; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; + import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; -import org.apache.hudi.avro.model.HoodieCompactionOperation; -import org.apache.hudi.avro.model.HoodieCompactionPlan; -import org.apache.hudi.config.HoodieWriteConfig; /** * UnBoundedPartitionAwareCompactionStrategy is a custom UnBounded Strategy. This will filter all the partitions that diff --git a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java index 41f79b0b5..ca63ee2a6 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java @@ -18,8 +18,9 @@ package org.apache.hudi.io.storage; -import org.apache.hadoop.conf.Configuration; import org.apache.hudi.avro.HoodieAvroWriteSupport; + +import org.apache.hadoop.conf.Configuration; import org.apache.parquet.hadoop.metadata.CompressionCodecName; public class HoodieParquetConfig { diff --git a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java index 9e9ccbc12..fa7b16528 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java @@ -18,23 +18,25 @@ package org.apache.hudi.io.storage; -import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieAvroWriteSupport; import org.apache.hudi.common.io.storage.HoodieWrapperFileSystem; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.HoodieAvroUtils; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.spark.TaskContext; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + /** * HoodieParquetWriter extends the ParquetWriter to help limit the size of underlying file. Provides a way to check if * the current file can take more records with the canWrite() diff --git a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriter.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriter.java index 322544150..d8cdb0f22 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriter.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriter.java @@ -18,10 +18,12 @@ package org.apache.hudi.io.storage; -import java.io.IOException; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.avro.generic.IndexedRecord; + +import java.io.IOException; + public interface HoodieStorageWriter { void writeAvroWithMetadata(R newRecord, HoodieRecord record) throws IOException; diff --git a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriterFactory.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriterFactory.java index 6624dc8f9..e179572b2 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriterFactory.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriterFactory.java @@ -18,21 +18,23 @@ package org.apache.hudi.io.storage; -import static org.apache.hudi.common.model.HoodieFileFormat.HOODIE_LOG; -import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET; - -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieAvroWriteSupport; import org.apache.hudi.common.BloomFilter; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; + +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.apache.parquet.avro.AvroSchemaConverter; +import java.io.IOException; + +import static org.apache.hudi.common.model.HoodieFileFormat.HOODIE_LOG; +import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET; + public class HoodieStorageWriterFactory { public static HoodieStorageWriter getStorageWriter( diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java b/hudi-client/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java index 916998560..d0f61f218 100644 --- a/hudi-client/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java @@ -18,11 +18,12 @@ package org.apache.hudi.metrics; -import com.codahale.metrics.Timer; -import com.google.common.annotations.VisibleForTesting; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.config.HoodieWriteConfig; + +import com.codahale.metrics.Timer; +import com.google.common.annotations.VisibleForTesting; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java b/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java index fb970e1d9..df9d9cc6c 100644 --- a/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java @@ -18,15 +18,17 @@ package org.apache.hudi.metrics; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; + import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; import com.google.common.io.Closeables; -import java.io.Closeable; -import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.Closeable; + /** * This is the main class of the metrics system. */ diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java index f0793b1b0..171174a49 100644 --- a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java @@ -18,16 +18,18 @@ package org.apache.hudi.metrics; +import org.apache.hudi.config.HoodieWriteConfig; + import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.graphite.Graphite; import com.codahale.metrics.graphite.GraphiteReporter; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.Closeable; import java.net.InetSocketAddress; import java.util.concurrent.TimeUnit; -import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * Implementation of Graphite reporter, which connects to the Graphite server, and send metrics to that server. diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index 10aa33f45..195d14737 100644 --- a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -18,8 +18,9 @@ package org.apache.hudi.metrics; -import com.codahale.metrics.MetricRegistry; import org.apache.hudi.config.HoodieWriteConfig; + +import com.codahale.metrics.MetricRegistry; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java index e8c0f1db4..b98c5d2a9 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java @@ -18,23 +18,6 @@ package org.apache.hudi.table; -import com.google.common.hash.Hashing; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.Serializable; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.WriteStatus; import org.apache.hudi.avro.model.HoodieActionInstant; import org.apache.hudi.avro.model.HoodieCleanerPlan; @@ -69,6 +52,12 @@ import org.apache.hudi.func.SparkBoundedInMemoryExecutor; import org.apache.hudi.io.HoodieCleanHelper; import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.io.HoodieMergeHandle; + +import com.google.common.hash.Hashing; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.parquet.avro.AvroParquetReader; @@ -78,6 +67,20 @@ import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.PairFlatMapFunction; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java index 185681c4a..e488e9f35 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java @@ -18,18 +18,6 @@ package org.apache.hudi.table; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; import org.apache.hudi.WriteStatus; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.HoodieRollbackStat; @@ -53,12 +41,26 @@ import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.func.MergeOnReadLazyInsertIterable; import org.apache.hudi.io.HoodieAppendHandle; import org.apache.hudi.io.compact.HoodieRealtimeTableCompactor; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + /** * Implementation of a more real-time read-optimized Hoodie Table where *

diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java index 09007b984..1ea174d46 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -18,18 +18,6 @@ package org.apache.hudi.table; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.WriteStatus; import org.apache.hudi.avro.model.HoodieCleanerPlan; import org.apache.hudi.avro.model.HoodieCompactionPlan; @@ -60,12 +48,26 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieSavepointException; import org.apache.hudi.index.HoodieIndex; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * Abstract implementation of a HoodieTable */ diff --git a/hudi-client/src/main/java/org/apache/hudi/table/RollbackExecutor.java b/hudi-client/src/main/java/org/apache/hudi/table/RollbackExecutor.java index 43ca9a44f..8c8455af0 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/RollbackExecutor.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/RollbackExecutor.java @@ -18,18 +18,6 @@ package org.apache.hudi.table; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import java.io.IOException; -import java.io.Serializable; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.PathFilter; import org.apache.hudi.common.HoodieRollbackStat; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -42,9 +30,24 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieRollbackException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.PathFilter; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; + +import java.io.IOException; +import java.io.Serializable; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/table/UserDefinedBulkInsertPartitioner.java b/hudi-client/src/main/java/org/apache/hudi/table/UserDefinedBulkInsertPartitioner.java index 420f87994..411918acd 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/UserDefinedBulkInsertPartitioner.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/UserDefinedBulkInsertPartitioner.java @@ -20,6 +20,7 @@ package org.apache.hudi.table; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; + import org.apache.spark.api.java.JavaRDD; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java index 9590e7e06..31d2e1dae 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java @@ -18,15 +18,18 @@ package org.apache.hudi.table; -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.Option; + import org.apache.spark.api.java.JavaRDD; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + import scala.Tuple2; /** diff --git a/hudi-client/src/main/java/org/apache/hudi/table/WorkloadStat.java b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadStat.java index ae759fa43..f69e59901 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/WorkloadStat.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadStat.java @@ -18,11 +18,12 @@ package org.apache.hudi.table; -import java.io.Serializable; -import java.util.HashMap; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.util.collection.Pair; +import java.io.Serializable; +import java.util.HashMap; + /** * Wraps stats about a single partition path. */ diff --git a/hudi-client/src/test/java/HoodieClientExample.java b/hudi-client/src/test/java/HoodieClientExample.java index 9e42db1f1..c483ca79f 100644 --- a/hudi-client/src/test/java/HoodieClientExample.java +++ b/hudi-client/src/test/java/HoodieClientExample.java @@ -16,12 +16,6 @@ * limitations under the License. */ -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.HoodieWriteClient; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.HoodieClientTestUtils; @@ -37,12 +31,20 @@ import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex.IndexType; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import java.util.ArrayList; +import java.util.List; + /** * Driver program that uses the Hoodie client with synthetic workload, and performs basic operations. *

diff --git a/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java b/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java index d23b15000..3e5f4983b 100644 --- a/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java +++ b/hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java @@ -17,17 +17,6 @@ package org.apache.hudi; -import java.io.IOException; -import java.io.Serializable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.HoodieTestDataGenerator; @@ -35,11 +24,24 @@ import org.apache.hudi.common.minicluster.HdfsTestService; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.FSUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.Serializable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + /** * The test harness for resource initialization and cleanup. */ diff --git a/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java index fcfbd736e..5a91ca164 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java @@ -18,19 +18,6 @@ package org.apache.hudi; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.HoodieClientTestUtils; @@ -58,10 +45,25 @@ import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaRDD; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * Test Cases for Async Compaction and Ingestion interaction */ diff --git a/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java index 021f13d00..8dfcd2413 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java @@ -18,29 +18,6 @@ package org.apache.hudi; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; @@ -72,12 +49,37 @@ import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * Test Cleaning related logic */ diff --git a/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java index f5523eab5..28d2351e8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java @@ -18,14 +18,6 @@ package org.apache.hudi; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.util.List; -import java.util.stream.Collectors; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.model.HoodieDataFile; @@ -41,9 +33,19 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieRollbackException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.api.java.JavaRDD; import org.junit.Test; +import java.io.File; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Test Cases for rollback of snapshots and commits */ diff --git a/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java index a987f8bd5..7d23ecde4 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java @@ -18,15 +18,6 @@ package org.apache.hudi; -import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.hudi.CompactionAdminClient.ValidationOpResult; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.HoodieLogFile; @@ -39,11 +30,22 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; + import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ; + public class TestCompactionAdminClient extends TestHoodieClientBase { private HoodieTableMetaClient metaClient; diff --git a/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java b/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java index e5520dfba..8344948f2 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java @@ -18,17 +18,19 @@ package org.apache.hudi; -import java.util.Arrays; -import java.util.concurrent.TimeoutException; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.util.ConsistencyGuard; import org.apache.hudi.common.util.ConsistencyGuardConfig; import org.apache.hudi.common.util.FailSafeConsistencyGuard; + +import org.apache.hadoop.fs.Path; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.Arrays; +import java.util.concurrent.TimeoutException; + public class TestConsistencyGuard extends HoodieClientTestHarness { @Before diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java index 5200a05d8..7e99c9ce5 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java @@ -18,19 +18,6 @@ package org.apache.hudi; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.HoodieCleanStat; import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.HoodieTestDataGenerator; @@ -56,6 +43,9 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.metrics.HoodieMetrics; import org.apache.hudi.table.HoodieTable; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -63,6 +53,18 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * Base Class providing setup/cleanup and utility methods for testing Hoodie Client facing tests */ diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java index 724c3fa29..790f1b322 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java @@ -18,29 +18,6 @@ package org.apache.hudi; -import static org.apache.hudi.common.HoodieTestDataGenerator.NULL_SCHEMA; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; -import static org.apache.hudi.common.util.ParquetUtils.readRowKeysFromParquet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.FileInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieCommitMetadata; @@ -67,10 +44,35 @@ import org.apache.hudi.exception.HoodieCommitException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaRDD; import org.junit.Assert; import org.junit.Test; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.HoodieTestDataGenerator.NULL_SCHEMA; +import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.common.util.ParquetUtils.readRowKeysFromParquet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + @SuppressWarnings("unchecked") public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java index 24e7ae4ff..858e5ff59 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java @@ -18,18 +18,20 @@ package org.apache.hudi; -import static org.junit.Assert.assertTrue; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; + +import org.apache.spark.api.java.JavaRDD; +import org.junit.Assert; +import org.junit.Test; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.spark.api.java.JavaRDD; -import org.junit.Assert; -import org.junit.Test; + +import static org.junit.Assert.assertTrue; @SuppressWarnings("unchecked") /** diff --git a/hudi-client/src/test/java/org/apache/hudi/TestMultiFS.java b/hudi-client/src/test/java/org/apache/hudi/TestMultiFS.java index 8dac99b5e..413483237 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestMultiFS.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestMultiFS.java @@ -18,10 +18,6 @@ package org.apache.hudi; -import static org.junit.Assert.assertEquals; - -import java.util.List; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieAvroPayload; @@ -35,6 +31,8 @@ import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; + +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -44,6 +42,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.List; + +import static org.junit.Assert.assertEquals; + public class TestMultiFS extends HoodieClientTestHarness { private static Logger logger = LogManager.getLogger(TestMultiFS.class); diff --git a/hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java b/hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java index f7f949266..0f1647b2d 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java @@ -18,14 +18,16 @@ package org.apache.hudi; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; import org.apache.hudi.common.model.HoodieRecord; + import org.junit.Test; import org.mockito.Mockito; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class TestWriteStatus { @Test public void testFailureFraction() throws IOException { diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java index 32aa5942c..f6cf940be 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java @@ -18,23 +18,6 @@ package org.apache.hudi.common; -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.HoodieReadClient; import org.apache.hudi.WriteStatus; import org.apache.hudi.avro.HoodieAvroWriteSupport; @@ -54,6 +37,11 @@ import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.io.storage.HoodieParquetConfig; import org.apache.hudi.io.storage.HoodieParquetWriter; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.parquet.avro.AvroSchemaConverter; @@ -65,6 +53,20 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SQLContext; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + /** * Utility methods to aid testing inside the HoodieClient module. */ diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java index 91e454bef..facc96473 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java @@ -18,11 +18,10 @@ package org.apache.hudi.common; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; @@ -34,9 +33,12 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.apache.hudi.common.model.HoodieTestUtils; -import org.apache.hudi.common.util.HoodieAvroUtils; -import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; /** * Utility methods to aid in testing MergeOnRead (workaround for HoodieReadClient for MOR) diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java index 52aae1be7..5ac436037 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java @@ -18,28 +18,6 @@ package org.apache.hudi.common; -import java.io.IOException; -import java.io.Serializable; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieCommitMetadata; @@ -56,6 +34,30 @@ import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + /** * Class to be used in tests to keep generating test inserts and updates against a corpus. *

diff --git a/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java b/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java index d30b99119..15fc3c332 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java @@ -18,7 +18,17 @@ package org.apache.hudi.common; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.MercifulJsonConverter; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.FileIOUtils; +import org.apache.hudi.common.util.Option; + import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -29,14 +39,6 @@ import java.util.Map.Entry; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.WriteStatus; -import org.apache.hudi.avro.MercifulJsonConverter; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieRecordPayload; -import org.apache.hudi.common.util.FileIOUtils; -import org.apache.hudi.common.util.Option; /** * Example row change event based on some example data used by testcases. The data avro schema is diff --git a/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java index 06bffe85d..e2d2a6c28 100644 --- a/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java +++ b/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java @@ -18,17 +18,19 @@ package org.apache.hudi.config; -import static org.junit.Assert.assertEquals; +import org.apache.hudi.config.HoodieWriteConfig.Builder; import com.google.common.collect.Maps; +import org.junit.Test; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Date; import java.util.Map; import java.util.Properties; -import org.apache.hudi.config.HoodieWriteConfig.Builder; -import org.junit.Test; + +import static org.junit.Assert.assertEquals; public class TestHoodieWriteConfig { diff --git a/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java index fbbe67fd0..e042e040b 100644 --- a/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java @@ -18,12 +18,6 @@ package org.apache.hudi.func; -import static org.apache.hudi.func.CopyOnWriteLazyInsertIterable.getTransformFunction; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieRecord; @@ -32,12 +26,21 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.func.CopyOnWriteLazyInsertIterable.HoodieInsertValueGenResult; + +import org.apache.avro.generic.IndexedRecord; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; + +import java.util.List; + import scala.Tuple2; +import static org.apache.hudi.func.CopyOnWriteLazyInsertIterable.getTransformFunction; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class TestBoundedInMemoryExecutor extends HoodieClientTestHarness { private final String commitTime = HoodieActiveTimeline.createNewCommitTime(); diff --git a/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java index ee4dc5859..5331c1525 100644 --- a/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java @@ -18,22 +18,6 @@ package org.apache.hudi.func; -import static org.apache.hudi.func.CopyOnWriteLazyInsertIterable.getTransformFunction; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.Semaphore; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieRecord; @@ -48,12 +32,31 @@ import org.apache.hudi.common.util.queue.FunctionBasedQueueProducer; import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.func.CopyOnWriteLazyInsertIterable.HoodieInsertValueGenResult; + +import org.apache.avro.generic.IndexedRecord; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import scala.Tuple2; +import static org.apache.hudi.func.CopyOnWriteLazyInsertIterable.getTransformFunction; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class TestBoundedInMemoryQueue extends HoodieClientTestHarness { private final String commitTime = HoodieActiveTimeline.createNewCommitTime(); diff --git a/hudi-client/src/test/java/org/apache/hudi/func/TestParquetReaderIterator.java b/hudi-client/src/test/java/org/apache/hudi/func/TestParquetReaderIterator.java index ee17d1e71..f84bd3e97 100644 --- a/hudi-client/src/test/java/org/apache/hudi/func/TestParquetReaderIterator.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestParquetReaderIterator.java @@ -18,15 +18,17 @@ package org.apache.hudi.func; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; import org.apache.hudi.exception.HoodieIOException; + import org.apache.parquet.hadoop.ParquetReader; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class TestParquetReaderIterator { @Test diff --git a/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java b/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java index aa50953d5..125e63efe 100644 --- a/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java @@ -18,15 +18,6 @@ package org.apache.hudi.func; -import static org.junit.Assert.fail; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.SerializableConfiguration; @@ -43,12 +34,23 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.io.HoodieMergeHandle; import org.apache.hudi.table.HoodieCopyOnWriteTable; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.parquet.avro.AvroReadSupport; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.fail; + public class TestUpdateMapFunction extends HoodieClientTestHarness { @Before diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java index f1a9af4d1..4d3f71859 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java @@ -18,8 +18,6 @@ package org.apache.hudi.index; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; @@ -31,6 +29,9 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; import org.apache.hudi.index.hbase.HBaseIndex; import org.apache.hudi.index.hbase.HBaseIndexQPSResourceAllocator; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java index 8fd294336..0b3e3f8f3 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java @@ -18,23 +18,6 @@ package org.apache.hudi.index; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Mockito.atMost; -import static org.mockito.Mockito.times; - -import java.util.Arrays; -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.HoodieWriteClient; import org.apache.hudi.WriteStatus; @@ -52,6 +35,16 @@ import org.apache.hudi.index.hbase.HBaseIndex; import org.apache.hudi.index.hbase.HBaseIndex.HbasePutBatchSizeCalculator; import org.apache.hudi.index.hbase.HBaseIndexQPSResourceAllocator; import org.apache.hudi.table.HoodieTable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.spark.api.java.JavaRDD; import org.junit.After; import org.junit.AfterClass; @@ -62,8 +55,18 @@ import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.List; + import scala.Tuple2; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.atMost; +import static org.mockito.Mockito.times; + /** * Note :: HBaseTestingUtility is really flaky with issues where the HbaseMiniCluster fails to shutdown across tests, * (see one problem here : https://issues.apache .org/jira/browse/HBASE-15835). Hence, the need to use diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index 8ca90805b..ebfb8cfa5 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -18,18 +18,19 @@ package org.apache.hudi.index; -import static org.junit.Assert.assertTrue; - import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.config.HoodieHBaseIndexConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.bloom.HoodieBloomIndex; import org.apache.hudi.index.hbase.HBaseIndex; + import org.junit.After; import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertTrue; + public class TestHoodieIndex extends HoodieClientTestHarness { @Before diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java index 22bb21861..5a02adc88 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java @@ -18,16 +18,18 @@ package org.apache.hudi.index.bloom; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import org.apache.hudi.common.util.collection.Pair; + +import org.junit.Test; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.apache.hudi.common.util.collection.Pair; -import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; public class TestBucketizedBloomCheckPartitioner { diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java index d8be596fa..3029dd72b 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java @@ -18,26 +18,6 @@ package org.apache.hudi.index.bloom; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import com.google.common.collect.Lists; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.Path; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.common.BloomFilter; import org.apache.hudi.common.HoodieClientTestUtils; @@ -54,6 +34,10 @@ import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.HoodieKeyLookupHandle; import org.apache.hudi.table.HoodieTable; + +import com.google.common.collect.Lists; +import org.apache.avro.Schema; +import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.junit.After; @@ -61,8 +45,27 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + import scala.Tuple2; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + @RunWith(Parameterized.class) public class TestHoodieBloomIndex extends HoodieClientTestHarness { diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java index 5da5332c7..d140c547a 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java @@ -18,22 +18,6 @@ package org.apache.hudi.index.bloom; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Lists; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.avro.Schema; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.TestRawTripPayload; @@ -46,13 +30,32 @@ import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; + +import com.google.common.collect.Lists; +import org.apache.avro.Schema; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.junit.After; import org.junit.Before; import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import scala.Tuple2; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { private String schemaStr; diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java index ba7400398..496e52b30 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java @@ -18,8 +18,7 @@ package org.apache.hudi.index.bloom; -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; +import org.junit.Test; import java.util.Collections; import java.util.HashMap; @@ -27,7 +26,9 @@ import java.util.HashSet; import java.util.Map; import java.util.Random; import java.util.UUID; -import org.junit.Test; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertTrue; /** * Tests {@link KeyRangeLookupTree} diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java index 1f1d32432..a672b91a8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java @@ -18,20 +18,6 @@ package org.apache.hudi.io; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Sets; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; import org.apache.hudi.common.HoodieTestDataGenerator; @@ -47,10 +33,26 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; + +import com.google.common.collect.Sets; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness { private Configuration hadoopConf; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java index ea2631748..7f2c6b484 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java @@ -18,12 +18,6 @@ package org.apache.hudi.io; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.List; -import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.HoodieWriteClient; import org.apache.hudi.WriteStatus; @@ -44,11 +38,19 @@ import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.bloom.HoodieBloomIndex; import org.apache.hudi.table.HoodieTable; + +import org.apache.hadoop.conf.Configuration; import org.apache.spark.api.java.JavaRDD; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class TestHoodieCompactor extends HoodieClientTestHarness { private Configuration hadoopConf; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java index 664063067..6a46592ea 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java @@ -18,13 +18,6 @@ package org.apache.hudi.io; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; - -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.HoodieWriteClient; import org.apache.hudi.WriteStatus; @@ -42,6 +35,8 @@ import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; + +import org.apache.hadoop.fs.FileSystem; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -50,6 +45,13 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; + @SuppressWarnings("unchecked") public class TestHoodieMergeHandle extends HoodieClientTestHarness { diff --git a/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java b/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java index 88a67ab45..90759778b 100755 --- a/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java @@ -18,18 +18,20 @@ package org.apache.hudi.io.storage; -import static org.junit.Assert.fail; - -import java.io.IOException; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.TestHoodieClientBase; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; + +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; + +import static org.junit.Assert.fail; + /** * Tests for {@link HoodieStorageWriterFactory}. */ diff --git a/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java index 93513a8ec..aa100b190 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java @@ -18,19 +18,6 @@ package org.apache.hudi.io.strategy; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.stream.Collectors; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.common.model.HoodieDataFile; import org.apache.hudi.common.model.HoodieLogFile; @@ -44,9 +31,24 @@ import org.apache.hudi.io.compact.strategy.DayBasedCompactionStrategy; import org.apache.hudi.io.compact.strategy.LogFileSizeBasedCompactionStrategy; import org.apache.hudi.io.compact.strategy.UnBoundedCompactionStrategy; import org.apache.hudi.io.compact.strategy.UnBoundedPartitionAwareCompactionStrategy; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class TestHoodieCompactionStrategy { private static final long MB = 1024 * 1024L; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieDataFile.java b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieDataFile.java index 62a9fa791..8d7851362 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieDataFile.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieDataFile.java @@ -18,9 +18,10 @@ package org.apache.hudi.io.strategy; -import java.util.UUID; import org.apache.hudi.common.model.HoodieDataFile; +import java.util.UUID; + public class TestHoodieDataFile extends HoodieDataFile { private final long size; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java index c8b5df93d..ec757b12d 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java @@ -18,9 +18,10 @@ package org.apache.hudi.io.strategy; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hadoop.fs.Path; + public class TestHoodieLogFile extends HoodieLogFile { private final long size; diff --git a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java index a99c6146f..e69afb80c 100644 --- a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java +++ b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java @@ -18,15 +18,16 @@ package org.apache.hudi.metrics; +import org.apache.hudi.config.HoodieWriteConfig; + +import org.junit.Before; +import org.junit.Test; + import static org.apache.hudi.metrics.Metrics.registerGauge; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.apache.hudi.config.HoodieWriteConfig; -import org.junit.Before; -import org.junit.Test; - public class TestHoodieMetrics { private HoodieMetrics metrics = null; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java index f9310d39a..686a72401 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java @@ -18,19 +18,6 @@ package org.apache.hudi.table; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.WriteStatus; import org.apache.hudi.common.BloomFilter; @@ -54,6 +41,9 @@ import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.table.HoodieCopyOnWriteTable.UpsertPartitioner; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.parquet.avro.AvroReadSupport; @@ -63,8 +53,21 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; + import scala.Tuple2; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class TestCopyOnWriteTable extends HoodieClientTestHarness { protected static Logger log = LogManager.getLogger(TestCopyOnWriteTable.class); diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java index a2424588d..07a91af73 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java @@ -18,12 +18,13 @@ package org.apache.hudi.table; -import static org.apache.hudi.common.model.HoodieTestUtils.generateFakeHoodieWriteStat; -import static org.apache.hudi.table.HoodieCopyOnWriteTable.averageBytesPerRecord; -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; + +import org.junit.Test; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -31,12 +32,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; -import org.apache.hudi.common.model.HoodieCommitMetadata; -import org.apache.hudi.common.model.HoodieWriteStat; -import org.apache.hudi.common.table.HoodieTimeline; -import org.apache.hudi.common.table.timeline.HoodieInstant; -import org.apache.hudi.common.util.Option; -import org.junit.Test; + +import static org.apache.hudi.common.model.HoodieTestUtils.generateFakeHoodieWriteStat; +import static org.apache.hudi.table.HoodieCopyOnWriteTable.averageBytesPerRecord; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TestHoodieRecordSizing { diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java index 152ad0f8d..b9ff1af98 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java @@ -18,23 +18,6 @@ package org.apache.hudi.table; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.HoodieClientTestHarness; import org.apache.hudi.HoodieReadClient; import org.apache.hudi.HoodieWriteClient; @@ -69,6 +52,10 @@ import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex.IndexType; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaRDD; import org.junit.After; import org.junit.Assert; @@ -76,6 +63,21 @@ import org.junit.Before; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + public class TestMergeOnReadTable extends HoodieClientTestHarness { @Before