diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java index f5a427fe2..392cd2329 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java @@ -18,13 +18,15 @@ package org.apache.hudi.avro; -import java.util.HashMap; -import org.apache.avro.Schema; import org.apache.hudi.common.BloomFilter; + +import org.apache.avro.Schema; import org.apache.parquet.avro.AvroWriteSupport; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.schema.MessageType; +import java.util.HashMap; + /** * Wrap AvroWriterSupport for plugging in the bloom filter. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java index 9cebe5127..c57dc84ad 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java @@ -18,21 +18,23 @@ package org.apache.hudi.avro; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Type; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.common.util.collection.Pair; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIOException; /** * Converts Json record to Avro Generic Record diff --git a/hudi-common/src/main/java/org/apache/hudi/common/BloomFilter.java b/hudi-common/src/main/java/org/apache/hudi/common/BloomFilter.java index 57ab2d8b9..06cc8b96d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/BloomFilter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/BloomFilter.java @@ -18,16 +18,19 @@ package org.apache.hudi.common; +import org.apache.hudi.exception.HoodieIndexException; + +import org.apache.hadoop.util.bloom.Key; +import org.apache.hadoop.util.hash.Hash; + +import javax.xml.bind.DatatypeConverter; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import javax.xml.bind.DatatypeConverter; -import org.apache.hadoop.util.bloom.Key; -import org.apache.hadoop.util.hash.Hash; -import org.apache.hudi.exception.HoodieIndexException; /** * A Bloom filter implementation built on top of {@link org.apache.hadoop.util.bloom.BloomFilter}. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java index 59f622f29..17c009966 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java @@ -18,12 +18,13 @@ package org.apache.hudi.common; -import java.io.Serializable; -import java.util.List; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; +import java.io.Serializable; +import java.util.List; + /** * Collects stats about a single partition clean operation */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java index e96222bab..b8ad4b543 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java @@ -18,21 +18,23 @@ package org.apache.hudi.common; +import org.apache.hudi.avro.MercifulJsonConverter; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.FileIOUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.avro.MercifulJsonConverter; -import org.apache.hudi.common.model.HoodieRecordPayload; -import org.apache.hudi.common.util.FileIOUtils; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.exception.HoodieException; public class HoodieJsonPayload implements HoodieRecordPayload { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java index e41475b51..5c489dfad 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java @@ -18,11 +18,12 @@ package org.apache.hudi.common; +import org.apache.hadoop.fs.FileStatus; + import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.apache.hadoop.fs.FileStatus; /** * Collects stats about a single partition clean operation diff --git a/hudi-common/src/main/java/org/apache/hudi/common/SerializableConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/SerializableConfiguration.java index acd17193f..40e5e316b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/SerializableConfiguration.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/SerializableConfiguration.java @@ -18,11 +18,12 @@ package org.apache.hudi.common; +import org.apache.hadoop.conf.Configuration; + import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; -import org.apache.hadoop.conf.Configuration; public class SerializableConfiguration implements Serializable { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/io/storage/HoodieWrapperFileSystem.java b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/HoodieWrapperFileSystem.java index 8b7a5f92b..5ddb837d3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/io/storage/HoodieWrapperFileSystem.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/HoodieWrapperFileSystem.java @@ -18,15 +18,13 @@ package org.apache.hudi.common.io.storage; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeoutException; +import org.apache.hudi.common.storage.StorageSchemes; +import org.apache.hudi.common.util.ConsistencyGuard; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.NoOpConsistencyGuard; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; @@ -51,12 +49,16 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; -import org.apache.hudi.common.storage.StorageSchemes; -import org.apache.hudi.common.util.ConsistencyGuard; -import org.apache.hudi.common.util.FSUtils; -import org.apache.hudi.common.util.NoOpConsistencyGuard; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIOException; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeoutException; /** * HoodieWrapperFileSystem wraps the default file system. It holds state about the open streams in the file system to diff --git a/hudi-common/src/main/java/org/apache/hudi/common/io/storage/SizeAwareFSDataOutputStream.java b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/SizeAwareFSDataOutputStream.java index 4da4b81fd..0211fb3cf 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/io/storage/SizeAwareFSDataOutputStream.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/SizeAwareFSDataOutputStream.java @@ -18,13 +18,15 @@ package org.apache.hudi.common.io.storage; +import org.apache.hudi.common.util.ConsistencyGuard; +import org.apache.hudi.exception.HoodieException; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; + import java.io.IOException; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.util.ConsistencyGuard; -import org.apache.hudi.exception.HoodieException; /** * Wrapper over FSDataOutputStream to keep track of the size of the written bytes. This gives a cheap way diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java b/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java index f9c39ab63..567c68a5c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java @@ -18,6 +18,12 @@ package org.apache.hudi.common.model; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; + +import org.apache.hadoop.fs.Path; + import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; @@ -25,10 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.avro.model.HoodieCompactionOperation; -import org.apache.hudi.common.util.FSUtils; -import org.apache.hudi.common.util.Option; /** * Encapsulates all the needed information about a compaction and make a decision whether this compaction is effective diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java index b9e122a1d..e12a29ce2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.util.Option; /** * Empty payload used for deletions diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java b/hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java index bb48b5a60..ae0221d73 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java @@ -18,11 +18,12 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import java.io.Serializable; import java.util.Objects; import java.util.TreeSet; import java.util.stream.Stream; -import org.apache.hudi.common.util.Option; /** * Within a file group, a slice is a combination of data file written at a commit time and list of log files, containing diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java index 4f9224342..b30947217 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java @@ -18,14 +18,16 @@ package org.apache.hudi.common.model; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; + +import java.io.IOException; + /** * This is a payload to wrap a existing Hoodie Avro Record. Useful to create a HoodieRecord over existing GenericRecords * in a hoodie datasets (useful in compactions) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java index fb70b37d6..c25bf49cb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java @@ -18,11 +18,16 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.FSUtils; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.IOException; import java.io.Serializable; import java.nio.charset.Charset; @@ -30,9 +35,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.hudi.common.util.FSUtils; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * All the metadata that gets stored along with a commit. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDataFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDataFile.java index 668fadde1..044d6434d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDataFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDataFile.java @@ -18,11 +18,13 @@ package org.apache.hudi.common.model; -import java.io.Serializable; -import java.util.Objects; +import org.apache.hudi.common.util.FSUtils; + import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.util.FSUtils; + +import java.io.Serializable; +import java.util.Objects; public class HoodieDataFile implements Serializable { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java index af1a9662f..9bc2467b9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java @@ -18,15 +18,16 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; + import java.io.Serializable; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.stream.Stream; -import org.apache.hudi.common.table.HoodieTimeline; -import org.apache.hudi.common.table.timeline.HoodieInstant; -import org.apache.hudi.common.util.Option; /** * A set of data/base files + set of log files, that make up an unit for all operations diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieKey.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieKey.java index 834589b4c..02f45d83e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieKey.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieKey.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.model; import com.google.common.base.Objects; + import java.io.Serializable; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java index 1b478ddba..29d26189d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java @@ -18,14 +18,16 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.FSUtils; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + import java.io.IOException; import java.io.Serializable; import java.util.Comparator; import java.util.Objects; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.util.FSUtils; /** * Abstracts a single log file. Contains methods to extract metadata like the fileId, version and extension from the log diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java index e967a5ddd..2ae8f11a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java @@ -18,16 +18,18 @@ package org.apache.hudi.common.model; -import java.io.IOException; -import java.util.Properties; +import org.apache.hudi.exception.HoodieException; + import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.util.Properties; + /** * The metadata that goes into the meta file in each partition */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java index eee8edad5..9aed0adeb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java @@ -18,11 +18,13 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import com.google.common.base.Objects; import com.google.common.collect.ImmutableList; + import java.io.Serializable; import java.util.List; -import org.apache.hudi.common.util.Option; /** * A Single Record managed by Hoodie TODO - Make this generic diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordLocation.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordLocation.java index b689b6e60..2c522d166 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordLocation.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordLocation.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.model; import com.google.common.base.Objects; + import java.io.Serializable; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java index 4839946c6..3e0384736 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java @@ -18,12 +18,14 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; + import java.io.IOException; import java.io.Serializable; import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.util.Option; /** * Every Hoodie dataset has an implementation of the HoodieRecordPayload This abstracts out callbacks which diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStat.java index 49c0db8a9..f86843115 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStat.java @@ -19,9 +19,11 @@ package org.apache.hudi.common.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import java.io.Serializable; + import javax.annotation.Nullable; +import java.io.Serializable; + @JsonIgnoreProperties(ignoreUnknown = true) public class HoodieRollingStat implements Serializable { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStatMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStatMetadata.java index f30a6157a..2818a1e4d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStatMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStatMetadata.java @@ -18,12 +18,13 @@ package org.apache.hudi.common.model; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.IOException; import java.io.Serializable; import java.util.HashMap; import java.util.Map; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * This class holds statistics about files belonging to a dataset diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java index 331dca847..786097118 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java @@ -20,10 +20,12 @@ package org.apache.hudi.common.model; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import java.io.Serializable; -import javax.annotation.Nullable; import org.apache.hadoop.fs.Path; +import javax.annotation.Nullable; + +import java.io.Serializable; + /** * Statistics about a single Hoodie write operation. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index ab1d8a6cb..a5f643a68 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -18,22 +18,24 @@ package org.apache.hudi.common.table; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.IOException; import java.io.Serializable; import java.util.Date; import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.model.HoodieAvroPayload; -import org.apache.hudi.common.model.HoodieFileFormat; -import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * Configurations on the Hoodie Table like type of ingestion, storage formats, hive table name etc Configurations are diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 882ce02eb..cd86b7ab9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -18,22 +18,6 @@ package org.apache.hudi.common.table; -import com.google.common.base.Preconditions; -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; import org.apache.hudi.common.SerializableConfiguration; import org.apache.hudi.common.io.storage.HoodieWrapperFileSystem; import org.apache.hudi.common.model.HoodieTableType; @@ -46,9 +30,27 @@ import org.apache.hudi.common.util.FailSafeConsistencyGuard; import org.apache.hudi.common.util.NoOpConsistencyGuard; import org.apache.hudi.exception.DatasetNotFoundException; import org.apache.hudi.exception.HoodieException; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + /** * HoodieTableMetaClient allows to access meta-data about a hoodie table It returns meta-data about * commits, savepoints, compactions, cleanups as a HoodieTimeline Create an instance of the diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java index ccc7e2e97..93496401e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java @@ -18,16 +18,17 @@ package org.apache.hudi.common.table; -import java.io.Serializable; -import java.util.function.BiPredicate; -import java.util.function.Predicate; -import java.util.stream.Stream; import org.apache.hudi.common.table.timeline.HoodieDefaultTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; +import java.io.Serializable; +import java.util.function.BiPredicate; +import java.util.function.Predicate; +import java.util.stream.Stream; + /** * HoodieTimeline is a view of meta-data instants in the hoodie dataset. Instants are specific points in time * represented as HoodieInstant. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableFileSystemView.java index 57f971d63..08828cca3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableFileSystemView.java @@ -18,8 +18,6 @@ package org.apache.hudi.common.table; -import java.util.List; -import java.util.stream.Stream; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieDataFile; @@ -28,6 +26,9 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import java.util.List; +import java.util.stream.Stream; + /** * Interface for viewing the table file system. * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java index 88bfe3957..bdd010edc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java @@ -18,23 +18,6 @@ package org.apache.hudi.common.table.log; -import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.INSTANT_TIME; -import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType.CORRUPT_BLOCK; - -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Deque; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; @@ -47,9 +30,28 @@ import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.util.SpillableMapUtils; import org.apache.hudi.exception.HoodieIOException; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.INSTANT_TIME; +import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType.CORRUPT_BLOCK; + /** * Implements logic to scan log blocks and expose valid and deleted log records to subclass implementation. Subclass is * free to either apply merging or expose raw data back to the caller. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java index e9118307a..897af5583 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java @@ -18,17 +18,6 @@ package org.apache.hudi.common.table.log; -import com.google.common.base.Preconditions; -import java.io.EOFException; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.BufferedFSInputStream; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSInputStream; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; @@ -42,9 +31,22 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.CorruptedLogFileException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieNotSupportedException; + +import com.google.common.base.Preconditions; +import org.apache.avro.Schema; +import org.apache.hadoop.fs.BufferedFSInputStream; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + /** * Scans a log file and provides block level iterator on the log file Loads the entire block contents in memory Can emit * either a DataBlock, CommandBlock, DeleteBlock or CorruptBlock (if one is found) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java index 7ca1cc080..0c04d74d4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java @@ -18,20 +18,22 @@ package org.apache.hudi.common.table.log; -import java.io.Closeable; -import java.io.IOException; -import java.util.Iterator; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; + +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + /** * File Format for Hoodie Log Files. The File Format consists of blocks each separated with a MAGIC sync marker. A Block * can either be a Data block, Command block or Delete Block. Data Block - Contains log records serialized as Avro diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java index c9dbafd2c..6128b2c72 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java @@ -18,17 +18,19 @@ package org.apache.hudi.common.table.log; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.exception.HoodieIOException; + +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + public class HoodieLogFormatReader implements HoodieLogFormat.Reader { private final List logFiles; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java index 17b5028c8..fe8e4a95b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java @@ -18,14 +18,6 @@ package org.apache.hudi.common.table.log; -import java.io.IOException; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; -import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.storage.StorageSchemes; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; @@ -34,9 +26,19 @@ import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; +import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; +import org.apache.hadoop.ipc.RemoteException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; + /** * HoodieLogFormatWriter can be used to append blocks to a log file Use HoodieLogFormat.WriterBuilder to construct */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java index a07d661b5..3ac121c11 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java @@ -18,12 +18,6 @@ package org.apache.hudi.common.table.log; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; @@ -33,9 +27,17 @@ import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.SpillableMapUtils; import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.exception.HoodieIOException; + +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + /** * Scans through all the blocks in a list of HoodieLogFile and builds up a compacted/merged list of records which will * be used as a lookup table when merging the base columnar file with the redo log file. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java index dc279952b..cdfe51670 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java @@ -18,13 +18,15 @@ package org.apache.hudi.common.table.log; -import java.util.List; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileSystem; + +import java.util.List; + public class HoodieUnMergedLogRecordScanner extends AbstractHoodieLogRecordScanner { private final LogRecordScannerCallback callback; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java index 985c394fe..2b30ab62c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java @@ -18,18 +18,13 @@ package org.apache.hudi.common.table.log.block; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.storage.SizeAwareDataInputStream; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; + import com.google.common.annotations.VisibleForTesting; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import javax.annotation.Nonnull; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; @@ -41,11 +36,19 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.storage.SizeAwareDataInputStream; -import org.apache.hudi.common.util.HoodieAvroUtils; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.exception.HoodieIOException; + +import javax.annotation.Nonnull; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; /** * DataBlock contains a list of records serialized using Avro. The Datablock contains 1. Data Block version 2. Total diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java index 2ac9cf6a3..75ca2cb14 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java @@ -18,12 +18,14 @@ package org.apache.hudi.common.table.log.block; -import java.util.HashMap; -import java.util.Map; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.util.Option; +import org.apache.hadoop.fs.FSDataInputStream; + +import java.util.HashMap; +import java.util.Map; + /** * Command block issues a specific command to the scanner */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java index d5e0c6cb2..87291ac6d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java @@ -18,12 +18,14 @@ package org.apache.hudi.common.table.log.block; -import java.io.IOException; -import java.util.Map; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.util.Option; +import org.apache.hadoop.fs.FSDataInputStream; + +import java.io.IOException; +import java.util.Map; + /** * Corrupt block is emitted whenever the scanner finds the length of the block written at the beginning does not match * (did not find a EOF or a sync marker after the length) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java index e4c2ff761..6b30372ac 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java @@ -18,6 +18,15 @@ package org.apache.hudi.common.table.log.block; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.storage.SizeAwareDataInputStream; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SerializationUtils; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.hadoop.fs.FSDataInputStream; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -25,13 +34,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hudi.common.model.HoodieKey; -import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.storage.SizeAwareDataInputStream; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SerializationUtils; -import org.apache.hudi.exception.HoodieIOException; /** * Delete block contains a list of keys to be deleted from scanning the blocks so far diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java index 3bf8fb21b..466952955 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java @@ -18,15 +18,6 @@ package org.apache.hudi.common.table.log.block; -import com.google.common.collect.Maps; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.EOFException; -import java.io.IOException; -import java.util.Map; -import javax.annotation.Nonnull; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.util.FSUtils; @@ -34,6 +25,19 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import com.google.common.collect.Maps; +import org.apache.hadoop.fs.FSDataInputStream; + +import javax.annotation.Nonnull; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.Map; + + /** * Abstract class defining a block in HoodieLogFile */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java index 721d55fbe..8c50ef1a4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java @@ -18,9 +18,22 @@ package org.apache.hudi.common.table.timeline; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.FileIOUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; + import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.IOException; import java.io.Serializable; import java.text.SimpleDateFormat; @@ -31,17 +44,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Stream; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.HoodieTimeline; -import org.apache.hudi.common.table.timeline.HoodieInstant.State; -import org.apache.hudi.common.util.FileIOUtils; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * Represents the Active Timeline for the HoodieDataset. Instants for the last 12 hours (configurable) is in the diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java index 552f8e7de..a58ff39ad 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java @@ -18,6 +18,17 @@ package org.apache.hudi.common.table.timeline; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.IOException; import java.io.Serializable; import java.util.Arrays; @@ -25,15 +36,6 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.HoodieTimeline; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * Represents the Archived Timeline for the HoodieDataset. Instants for the last 12 hours (configurable) is in the diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java index 72318dd0d..90067f27e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java @@ -18,9 +18,16 @@ package org.apache.hudi.common.table.timeline; -import static java.util.Collections.reverse; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.exception.HoodieException; import com.google.common.collect.Sets; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.List; @@ -29,13 +36,8 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.hudi.common.table.HoodieTimeline; -import org.apache.hudi.common.table.timeline.HoodieInstant.State; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.StringUtils; -import org.apache.hudi.exception.HoodieException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; + +import static java.util.Collections.reverse; /** * HoodieDefaultTimeline is a default implementation of the HoodieTimeline. It provides methods to inspect a diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java index 471d2d0d2..521f443d4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java @@ -18,12 +18,14 @@ package org.apache.hudi.common.table.timeline; -import java.io.Serializable; -import java.util.Objects; -import org.apache.hadoop.fs.FileStatus; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.util.FSUtils; +import org.apache.hadoop.fs.FileStatus; + +import java.io.Serializable; +import java.util.Objects; + /** * A Hoodie Instant represents a action done on a hoodie dataset. All actions start with a inflight instant and then * create a completed instant after done. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/CompactionOpDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/CompactionOpDTO.java index 5fc4d11af..b3782d9b2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/CompactionOpDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/CompactionOpDTO.java @@ -18,15 +18,17 @@ package org.apache.hudi.common.table.timeline.dto; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.hudi.common.model.CompactionOperation; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.collection.Pair; @JsonIgnoreProperties(ignoreUnknown = true) public class CompactionOpDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/DataFileDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/DataFileDTO.java index cb587147a..8ae94d67a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/DataFileDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/DataFileDTO.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.table.timeline.dto; +import org.apache.hudi.common.model.HoodieDataFile; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.hudi.common.model.HoodieDataFile; @JsonIgnoreProperties(ignoreUnknown = true) public class DataFileDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java index 632309704..48187928c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java @@ -20,10 +20,11 @@ package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.Serializable; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import java.io.Serializable; + /** * A serializable FS Permission */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileGroupDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileGroupDTO.java index 9b26352d1..815f76f2c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileGroupDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileGroupDTO.java @@ -18,13 +18,15 @@ package org.apache.hudi.common.table.timeline.dto; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.stream.Collectors; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.table.HoodieTableMetaClient; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.stream.Collectors; + @JsonIgnoreProperties(ignoreUnknown = true) public class FileGroupDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FilePathDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FilePathDTO.java index eae4f4aa2..aa67b8f9f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FilePathDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FilePathDTO.java @@ -20,9 +20,10 @@ package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.fs.Path; + import java.net.URI; import java.net.URISyntaxException; -import org.apache.hadoop.fs.Path; @JsonIgnoreProperties(ignoreUnknown = true) public class FilePathDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileSliceDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileSliceDTO.java index 5bfd3fbc3..251f2e15c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileSliceDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileSliceDTO.java @@ -18,11 +18,13 @@ package org.apache.hudi.common.table.timeline.dto; +import org.apache.hudi.common.model.FileSlice; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.List; import java.util.stream.Collectors; -import org.apache.hudi.common.model.FileSlice; @JsonIgnoreProperties(ignoreUnknown = true) public class FileSliceDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileStatusDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileStatusDTO.java index 79fa35c9a..59eccb353 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileStatusDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileStatusDTO.java @@ -18,11 +18,13 @@ package org.apache.hudi.common.table.timeline.dto; +import org.apache.hudi.exception.HoodieException; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.IOException; import org.apache.hadoop.fs.FileStatus; -import org.apache.hudi.exception.HoodieException; + +import java.io.IOException; @JsonIgnoreProperties(ignoreUnknown = true) public class FileStatusDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java index 20beec654..c0a1b3a1f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.table.timeline.dto; +import org.apache.hudi.common.table.timeline.HoodieInstant; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.hudi.common.table.timeline.HoodieInstant; @JsonIgnoreProperties(ignoreUnknown = true) public class InstantDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/LogFileDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/LogFileDTO.java index bb3345b46..2a15ba04c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/LogFileDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/LogFileDTO.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.table.timeline.dto; +import org.apache.hudi.common.model.HoodieLogFile; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.fs.FileStatus; -import org.apache.hudi.common.model.HoodieLogFile; @JsonIgnoreProperties(ignoreUnknown = true) public class LogFileDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/TimelineDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/TimelineDTO.java index e68ff0b8b..3fb875bcb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/TimelineDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/TimelineDTO.java @@ -18,14 +18,16 @@ package org.apache.hudi.common.table.timeline.dto; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.stream.Collectors; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieDefaultTimeline; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.stream.Collectors; + @JsonIgnoreProperties(ignoreUnknown = true) public class TimelineDTO { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java index b477f242a..45472a595 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java @@ -18,24 +18,6 @@ package org.apache.hudi.common.table.view; -import com.google.common.base.Preconditions; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieDataFile; @@ -52,9 +34,29 @@ import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * Common thread-safe implementation for multiple TableFileSystemView Implementations. Provides uniform handling of (a) * Loading file-system views from underlying file-system (b) Pending compaction operations and changing file-system diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java index 6a233a285..c2f49a43d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java @@ -18,15 +18,17 @@ package org.apache.hudi.common.table.view; -import java.util.concurrent.ConcurrentHashMap; import org.apache.hudi.common.SerializableConfiguration; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.SyncableFileSystemView; import org.apache.hudi.common.util.Functions.Function2; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.util.concurrent.ConcurrentHashMap; + /** * A container that can potentially hold one or more dataset's file-system views. There is one view for each dataset. * This is a view built against a timeline containing completed actions. In an embedded timeline-server mode, this diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java index ee556f036..bcaac83e2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java @@ -18,12 +18,14 @@ package org.apache.hudi.common.table.view; +import org.apache.hudi.config.DefaultHoodieConfig; + import com.google.common.base.Preconditions; + import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; -import org.apache.hudi.config.DefaultHoodieConfig; /** * File System View Storage Configurations diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java index c82ac2713..6ae6fa601 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java @@ -18,15 +18,6 @@ package org.apache.hudi.common.table.view; -import com.google.common.base.Preconditions; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieFileGroupId; @@ -35,9 +26,20 @@ import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.TableFileSystemView; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.FileStatus; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * TableFileSystemView Implementations based on in-memory storage. * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java index f31ac4de8..6dccb4b45 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java @@ -18,12 +18,6 @@ package org.apache.hudi.common.table.view; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieRestoreMetadata; @@ -43,9 +37,17 @@ import org.apache.hudi.common.util.TimelineDiffHelper; import org.apache.hudi.common.util.TimelineDiffHelper.TimelineDiffResult; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + /** * Adds the capability to incrementally sync the changes to file-system view as and when new instants gets completed. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java index 696d77921..e622cb167 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java @@ -18,9 +18,6 @@ package org.apache.hudi.common.table.view; -import java.io.Serializable; -import java.util.List; -import java.util.stream.Stream; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieDataFile; @@ -34,9 +31,14 @@ import org.apache.hudi.common.util.Functions.Function2; import org.apache.hudi.common.util.Functions.Function3; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.Serializable; +import java.util.List; +import java.util.stream.Stream; + /** * A file system view which proxies request to a preferred File System View implementation. In case of error, flip all * subsequent calls to a backup file-system view implementation. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java index d405cd2e7..3cac97e8e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java @@ -18,18 +18,6 @@ package org.apache.hudi.common.table.view; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import java.io.IOException; -import java.io.Serializable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.apache.http.client.fluent.Request; -import org.apache.http.client.fluent.Response; -import org.apache.http.client.utils.URIBuilder; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieDataFile; @@ -48,9 +36,23 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieRemoteException; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import org.apache.http.client.fluent.Request; +import org.apache.http.client.fluent.Response; +import org.apache.http.client.utils.URIBuilder; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + /** * A proxy for table file-system view which translates local View API calls to REST calls to remote timeline service */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java index 8580653c8..1dafb4a37 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java @@ -18,16 +18,6 @@ package org.apache.hudi.common.table.view; -import com.google.common.base.Preconditions; -import java.io.Serializable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieDataFile; @@ -40,9 +30,21 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.RocksDBDAO; import org.apache.hudi.common.util.RocksDBSchemaHelper; import org.apache.hudi.common.util.collection.Pair; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * A file-system view implementation on top of embedded Rocks DB store. For each DataSet : 3 column Family is added for * storing (1) File-Slices and Data Files for View lookups (2) Pending compaction operations (3) Partitions tracked diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java index ad31db4f5..6527da949 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java @@ -18,12 +18,6 @@ package org.apache.hudi.common.table.view; -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieFileGroupId; @@ -32,9 +26,17 @@ import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.util.DefaultSizeEstimator; import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.common.util.collection.Pair; + +import org.apache.hadoop.fs.FileStatus; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + /** * Table FileSystemView implementation where view is stored in spillable disk using fixed memory */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java index 1b60ffa75..1b5a7b796 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java @@ -18,22 +18,6 @@ package org.apache.hudi.common.util; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import org.apache.avro.file.DataFileReader; -import org.apache.avro.file.DataFileWriter; -import org.apache.avro.file.FileReader; -import org.apache.avro.file.SeekableByteArrayInput; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.specific.SpecificRecordBase; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata; import org.apache.hudi.avro.model.HoodieCleanerPlan; @@ -46,6 +30,24 @@ import org.apache.hudi.avro.model.HoodieSavepointPartitionMetadata; import org.apache.hudi.common.HoodieCleanStat; import org.apache.hudi.common.HoodieRollbackStat; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.file.FileReader; +import org.apache.avro.file.SeekableByteArrayInput; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecordBase; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + public class AvroUtils { private static final Integer DEFAULT_VERSION = 1; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java index 56fc4de84..66a989ca7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java @@ -18,13 +18,6 @@ package org.apache.hudi.common.util; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.model.CompactionOperation; @@ -38,9 +31,18 @@ import org.apache.hudi.common.versioning.compaction.CompactionPlanMigrator; import org.apache.hudi.common.versioning.compaction.CompactionV1MigrationHandler; import org.apache.hudi.common.versioning.compaction.CompactionV2MigrationHandler; import org.apache.hudi.exception.HoodieException; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * Helper class to generate compaction plan from FileGroup/FileSlice abstraction */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java index 89190aa45..eb3337112 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.util; +import org.apache.hadoop.fs.Path; + import java.io.IOException; import java.util.List; import java.util.concurrent.TimeoutException; -import org.apache.hadoop.fs.Path; /** * Ensures file create/delete operation is visible diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuardConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuardConfig.java index 761aabf89..eec18f1a5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuardConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuardConfig.java @@ -18,11 +18,12 @@ package org.apache.hudi.common.util; +import org.apache.hudi.config.DefaultHoodieConfig; + import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; -import org.apache.hudi.config.DefaultHoodieConfig; public class ConsistencyGuardConfig extends DefaultHoodieConfig { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/DFSPropertiesConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/util/DFSPropertiesConfiguration.java index 0d37a0f3f..5bcfe1f34 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/DFSPropertiesConfiguration.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/DFSPropertiesConfiguration.java @@ -18,15 +18,16 @@ package org.apache.hudi.common.util; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.HashSet; import java.util.Set; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * A simplified versions of Apache commons - PropertiesConfiguration, that supports limited field types and hierarchical diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java index 60d85382d..506daddce 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java @@ -18,8 +18,30 @@ package org.apache.hudi.common.util; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.InvalidHoodiePathException; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -32,26 +54,6 @@ import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hudi.common.model.HoodieFileFormat; -import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.model.HoodiePartitionMetadata; -import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.timeline.HoodieInstant; -import org.apache.hudi.common.util.collection.Pair; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.exception.InvalidHoodiePathException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * Utility functions related to accessing the file storage diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java index 2bcb901f3..88786066e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java @@ -19,6 +19,12 @@ package org.apache.hudi.common.util; import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -27,11 +33,6 @@ import java.util.List; import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * A consistency checker that fails if it is unable to meet the required condition within a specified timeout diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java index 875ba05dd..75ed689a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java @@ -18,6 +18,23 @@ package org.apache.hudi.common.util; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.SchemaCompatabilityException; + +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.node.NullNode; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -30,21 +47,6 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.exception.SchemaCompatabilityException; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.node.NullNode; /** * Helper class to do common stuff across Avro. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java index 12d435fd5..11be1c6c8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.util; -import org.apache.avro.Schema; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.collection.ExternalSpillableMap; + +import org.apache.avro.Schema; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieTimer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieTimer.java index d4206fccb..2b8fac851 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieTimer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieTimer.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.util; +import org.apache.hudi.exception.HoodieException; + import java.util.ArrayDeque; import java.util.Deque; -import org.apache.hudi.exception.HoodieException; /** * Timing utility to help keep track of execution times of code blocks. This class helps to allow multiple timers diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/LogReaderUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/LogReaderUtils.java index b572b931f..378a95324 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/LogReaderUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/LogReaderUtils.java @@ -18,13 +18,6 @@ package org.apache.hudi.common.util; -import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; @@ -35,6 +28,15 @@ import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobConf; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + /** * Utils class for performing various log file reading operations */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/NetworkUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/NetworkUtils.java index 1e82d719f..9598714a5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/NetworkUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/NetworkUtils.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.util; +import org.apache.hudi.exception.HoodieException; + import java.io.IOException; import java.net.ServerSocket; -import org.apache.hudi.exception.HoodieException; public class NetworkUtils { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/NoOpConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/util/NoOpConsistencyGuard.java index 463a1410c..bc3466d29 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/NoOpConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/NoOpConsistencyGuard.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.util; -import java.util.List; import org.apache.hadoop.fs.Path; +import java.util.List; + /** * Default Consistency guard that does nothing. Used for HDFS deployments */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java index b8b2aefe7..dc50c5671 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java @@ -22,6 +22,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.Sets; + import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.reflect.Array; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java index 7baf368e5..749c68d70 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java @@ -18,6 +18,25 @@ package org.apache.hudi.common.util; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.MetadataNotFoundException; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.avro.AvroReadSupport; +import org.apache.parquet.avro.AvroSchemaConverter; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.schema.MessageType; + import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -26,23 +45,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.avro.HoodieAvroWriteSupport; -import org.apache.hudi.common.BloomFilter; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.exception.MetadataNotFoundException; -import org.apache.parquet.avro.AvroParquetReader; -import org.apache.parquet.avro.AvroReadSupport; -import org.apache.parquet.avro.AvroSchemaConverter; -import org.apache.parquet.hadoop.ParquetFileReader; -import org.apache.parquet.hadoop.ParquetReader; -import org.apache.parquet.hadoop.metadata.ParquetMetadata; -import org.apache.parquet.schema.MessageType; /** * Utility functions involving with parquet. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java index c9125e35a..b50612692 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java @@ -18,16 +18,18 @@ package org.apache.hudi.common.util; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.exception.HoodieException; + import com.google.common.reflect.ClassPath; import com.google.common.reflect.ClassPath.ClassInfo; + import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; -import org.apache.hudi.common.model.HoodieRecordPayload; -import org.apache.hudi.exception.HoodieException; public class ReflectionUtils { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java index 578f19382..9453ec5d6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java @@ -18,21 +18,12 @@ package org.apache.hudi.common.util; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.rocksdb.ColumnFamilyDescriptor; @@ -48,6 +39,17 @@ import org.rocksdb.Statistics; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * Data access objects for storing and retrieving objects in Rocks DB. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBSchemaHelper.java b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBSchemaHelper.java index 437fe5b15..574021b68 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBSchemaHelper.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBSchemaHelper.java @@ -18,13 +18,14 @@ package org.apache.hudi.common.util; -import java.util.Arrays; -import java.util.List; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.table.HoodieTableMetaClient; +import java.util.Arrays; +import java.util.List; + /** * Helper class to generate Key and column names for rocksdb based view * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java index b686bd16b..7bc0f5fa3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java @@ -18,19 +18,21 @@ package org.apache.hudi.common.util; +import org.apache.hudi.exception.HoodieSerializationException; + import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Serializer; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.serializers.FieldSerializer; import com.esotericsoftware.reflectasm.ConstructorAccess; +import org.objenesis.instantiator.ObjectInstantiator; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import org.apache.hudi.exception.HoodieSerializationException; -import org.objenesis.instantiator.ObjectInstantiator; /** * {@link SerializationUtils} class internally uses {@link Kryo} serializer for serializing / deserializing objects. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java index 3b22893f3..da9524706 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java @@ -18,10 +18,6 @@ package org.apache.hudi.common.util; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.zip.CRC32; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; @@ -29,6 +25,12 @@ import org.apache.hudi.common.util.collection.DiskBasedMap.FileEntry; import org.apache.hudi.common.util.collection.io.storage.SizeAwareDataOutputStream; import org.apache.hudi.exception.HoodieCorruptedDataException; +import org.apache.avro.generic.GenericRecord; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.zip.CRC32; + public class SpillableMapUtils { /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java b/hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java index e4ca22191..21d462ccc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java @@ -18,17 +18,19 @@ package org.apache.hudi.common.util; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.util.collection.Pair; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + public class TimelineDiffHelper { protected static Logger log = LogManager.getLogger(TimelineDiffHelper.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskBasedMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskBasedMap.java index 46cac4a16..97b897bb9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskBasedMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskBasedMap.java @@ -18,6 +18,16 @@ package org.apache.hudi.common.util.collection; +import org.apache.hudi.common.util.SerializationUtils; +import org.apache.hudi.common.util.SpillableMapUtils; +import org.apache.hudi.common.util.collection.io.storage.SizeAwareDataOutputStream; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieNotSupportedException; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -36,14 +46,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; -import org.apache.hudi.common.util.SerializationUtils; -import org.apache.hudi.common.util.SpillableMapUtils; -import org.apache.hudi.common.util.collection.io.storage.SizeAwareDataOutputStream; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.exception.HoodieNotSupportedException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * This class provides a disk spillable only map implementation. All of the data is currenly written to one file, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java index 7f44b1056..b120c0823 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java @@ -18,6 +18,13 @@ package org.apache.hudi.common.util.collection; +import org.apache.hudi.common.util.ObjectSizeCalculator; +import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -29,11 +36,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Stream; -import org.apache.hudi.common.util.ObjectSizeCalculator; -import org.apache.hudi.common.util.SizeEstimator; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * An external map that spills content to disk when there is insufficient space for it to grow. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java index 27ffff86d..aaaf32e8e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java @@ -18,13 +18,14 @@ package org.apache.hudi.common.util.collection; +import org.apache.hudi.exception.HoodieException; + import java.io.IOException; import java.io.RandomAccessFile; import java.util.Iterator; import java.util.Map; import java.util.function.Consumer; import java.util.stream.Collectors; -import org.apache.hudi.exception.HoodieException; /** * Iterable to lazily fetch values spilled to disk. This class uses RandomAccessFile to randomly access the position of diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java index 1bb5065ad..0744b5052 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java @@ -18,13 +18,14 @@ package org.apache.hudi.common.util.collection; +import org.apache.hudi.common.util.RocksDBDAO; +import org.apache.hudi.exception.HoodieNotSupportedException; + import java.io.Serializable; import java.util.Collection; import java.util.Iterator; import java.util.Map; import java.util.Set; -import org.apache.hudi.common.util.RocksDBDAO; -import org.apache.hudi.exception.HoodieNotSupportedException; public final class RocksDBBasedMap implements Map { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java index e31999429..6e6797968 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java @@ -18,6 +18,14 @@ package org.apache.hudi.common.util.queue; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.exception.HoodieException; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -28,12 +36,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.hudi.common.util.DefaultSizeEstimator; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SizeEstimator; -import org.apache.hudi.exception.HoodieException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * Executor which orchestrates concurrent producers and consumers communicating through a bounded in-memory queue. This diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java index 79a1920fb..a8f06d244 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java @@ -18,8 +18,16 @@ package org.apache.hudi.common.util.queue; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.exception.HoodieException; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + import java.util.Iterator; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; @@ -28,12 +36,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; -import org.apache.hudi.common.util.DefaultSizeEstimator; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SizeEstimator; -import org.apache.hudi.exception.HoodieException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * Used for enqueueing input records. Queue limit is controlled by {@link #memoryLimit}. Unlike standard bounded queue diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/FunctionBasedQueueProducer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/FunctionBasedQueueProducer.java index 175dbf241..e6266f7e2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/FunctionBasedQueueProducer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/FunctionBasedQueueProducer.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.util.queue; -import java.util.function.Function; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.util.function.Function; + /** * Buffer producer which allows custom functions to insert entries to queue. * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/IteratorBasedQueueProducer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/IteratorBasedQueueProducer.java index 5d52012d6..a5948ac84 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/IteratorBasedQueueProducer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/IteratorBasedQueueProducer.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.util.queue; -import java.util.Iterator; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.util.Iterator; + /** * Iterator based producer which pulls entry from iterator and produces items for the queue * diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java index 2c2c5b58f..ebd25c84d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java @@ -18,12 +18,14 @@ package org.apache.hudi.common.versioning; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.collection.Pair; + import com.google.common.base.Preconditions; + import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.util.collection.Pair; /** * Migrates a specific metadata type stored in .hoodie folder to latest version diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionPlanMigrator.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionPlanMigrator.java index f8a4fb422..b2b963479 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionPlanMigrator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionPlanMigrator.java @@ -18,11 +18,12 @@ package org.apache.hudi.common.versioning.compaction; -import java.util.Arrays; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.versioning.MetadataMigrator; +import java.util.Arrays; + /** * Responsible for handling different versions of compaction plan */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java index 67cb21398..7db7916b2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java @@ -18,17 +18,19 @@ package org.apache.hudi.common.versioning.compaction; -import com.google.common.base.Preconditions; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.versioning.AbstractMigratorBase; +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.Path; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + /** * Compaction V1 has absolute paths as part of compaction operations */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java index e1d60ba6c..cc73a2f3d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java @@ -18,16 +18,18 @@ package org.apache.hudi.common.versioning.compaction; -import com.google.common.base.Preconditions; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.versioning.AbstractMigratorBase; +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.Path; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + /** * With version 2 of compaction plan, paths are no longer absolute. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/DatasetNotFoundException.java b/hudi-common/src/main/java/org/apache/hudi/exception/DatasetNotFoundException.java index 711516a1d..c4c0299f0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/exception/DatasetNotFoundException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/DatasetNotFoundException.java @@ -18,10 +18,11 @@ package org.apache.hudi.exception; -import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import java.io.IOException; + /** *

* Exception thrown to indicate that a hoodie dataset was not found on the path provided diff --git a/hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java b/hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java index fd087d0ba..3042710a4 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java @@ -18,16 +18,18 @@ package org.apache.hudi.common; -import java.io.IOException; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.SyncableFileSystemView; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; + import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import java.io.IOException; + public class HoodieCommonTestHarness { protected String basePath = null; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java b/hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java index 76e980568..bf28bc6eb 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java @@ -18,9 +18,10 @@ package org.apache.hudi.common; -import java.io.IOException; import org.junit.Test; +import java.io.IOException; + public class TestBloomFilter { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java index 3d48b11fe..f6d035a38 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java @@ -18,19 +18,21 @@ package org.apache.hudi.common.minicluster; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.util.FileIOUtils; + import com.google.common.base.Preconditions; import com.google.common.io.Files; -import java.io.File; -import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hudi.common.model.HoodieTestUtils; -import org.apache.hudi.common.util.FileIOUtils; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.File; +import java.io.IOException; + /** * An HDFS minicluster service implementation. */ diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java index a671d1db5..b9f7d6457 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java @@ -18,12 +18,13 @@ package org.apache.hudi.common.minicluster; -import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.zookeeper.server.ZooKeeperServer; +import java.io.IOException; + public class MiniClusterUtil { private static MiniDFSCluster dfsCluster; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java index 89992eecb..bac93ee48 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java @@ -20,6 +20,14 @@ package org.apache.hudi.common.minicluster; import com.google.common.base.Preconditions; import com.google.common.io.Files; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.persistence.FileTxnLog; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -28,13 +36,6 @@ import java.io.OutputStream; import java.io.Reader; import java.net.InetSocketAddress; import java.net.Socket; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -import org.apache.zookeeper.server.NIOServerCnxnFactory; -import org.apache.zookeeper.server.ZooKeeperServer; -import org.apache.zookeeper.server.persistence.FileTxnLog; /** * A Zookeeper minicluster service implementation. diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java index 546bf1464..4251adb24 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java @@ -18,14 +18,16 @@ package org.apache.hudi.common.model; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; + +import java.io.IOException; + public class AvroBinaryTestPayload implements HoodieRecordPayload { private final byte[] recordBytes; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java index 2a3ffab55..405ba51b8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java @@ -18,44 +18,6 @@ package org.apache.hudi.common.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; -import com.esotericsoftware.kryo.serializers.JavaSerializer; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.util.StringUtils; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.HoodieCleanStat; @@ -77,6 +39,46 @@ import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.esotericsoftware.kryo.serializers.JavaSerializer; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.util.StringUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + public class HoodieTestUtils { public static final String TEST_EXTENSION = ".test"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java index 8b3eb38ef..a020a5c71 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.model; -import java.util.List; import org.junit.Assert; import org.junit.Test; +import java.util.List; + public class TestHoodieCommitMetadata { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java index e9c32a160..7dd6da748 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java @@ -18,19 +18,20 @@ package org.apache.hudi.common.model; -import static org.junit.Assert.fail; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SchemaTestUtil; + +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SchemaTestUtil; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.fail; /** * Tests for {@link HoodieRecord}. diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java index 5c460f9c4..d2027409a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java @@ -18,16 +18,18 @@ package org.apache.hudi.common.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; + +import org.apache.hadoop.fs.Path; +import org.junit.Test; import java.text.SimpleDateFormat; import java.util.Date; import java.util.UUID; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.util.FSUtils; -import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class TestHoodieWriteStat { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java b/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java index 3e20fe60c..2a0aa1fc9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java @@ -18,12 +18,12 @@ package org.apache.hudi.common.storage; +import org.junit.Test; + import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.junit.Test; - public class TestStorageSchemes { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java index 68fb1c9e7..ab443707d 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java @@ -18,28 +18,30 @@ package org.apache.hudi.common.table; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Lists; -import java.io.IOException; -import java.util.stream.Collectors; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; + +import com.google.common.collect.Lists; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + public class TestHoodieTableMetaClient extends HoodieCommonTestHarness { @Before diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java index dffb74232..f716c0110 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java @@ -18,33 +18,6 @@ package org.apache.hudi.common.table.log; -import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import com.google.common.collect.Maps; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.minicluster.MiniClusterUtil; import org.apache.hudi.common.model.HoodieArchivedLogFile; @@ -67,6 +40,14 @@ import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.exception.CorruptedLogFileException; +import com.google.common.collect.Maps; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -75,6 +56,26 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + @SuppressWarnings("Duplicates") @RunWith(Parameterized.class) public class TestHoodieLogFormat extends HoodieCommonTestHarness { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java index cfa8f22ab..2de7d93ec 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java @@ -18,16 +18,14 @@ package org.apache.hudi.common.table.log; -import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; +import org.apache.hudi.common.minicluster.MiniClusterUtil; +import org.apache.hudi.common.model.HoodieArchivedLogFile; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.util.SchemaTestUtil; import com.google.common.collect.Maps; -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.TimeoutException; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -39,17 +37,21 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hudi.common.minicluster.MiniClusterUtil; -import org.apache.hudi.common.model.HoodieArchivedLogFile; -import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; -import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; -import org.apache.hudi.common.table.log.block.HoodieLogBlock; -import org.apache.hudi.common.util.SchemaTestUtil; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeoutException; + +import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; + /** * This class is intentionally using a different way of setting up the MiniDFSCluster and not relying on * {@link MiniClusterUtil} to reproduce append() issue : https://issues.apache.org/jira/browse/HDFS-6325 Reference : diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java index 72df1c02b..2d91d01a9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java @@ -18,14 +18,15 @@ package org.apache.hudi.common.table.string; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; + import java.io.IOException; import java.util.Comparator; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.hudi.common.table.HoodieTimeline; -import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; -import org.apache.hudi.common.table.timeline.HoodieInstant; public class MockHoodieTimeline extends HoodieActiveTimeline { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java index 202bc79f5..f4a4a4d66 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java @@ -18,24 +18,25 @@ package org.apache.hudi.common.table.string; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.stream.Stream; - import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; + import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.io.IOException; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + public class TestHoodieActiveTimeline extends HoodieCommonTestHarness { private HoodieActiveTimeline timeline; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java index eb1a15e89..26cea6817 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java @@ -18,25 +18,6 @@ package org.apache.hudi.common.table.view; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.CompactionOperation; @@ -57,12 +38,33 @@ import org.apache.hudi.common.util.CompactionUtils; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + @SuppressWarnings("ResultOfMethodCallIgnored") public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java index acc8cb416..adc860dfb 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java @@ -18,24 +18,6 @@ package org.apache.hudi.common.table.view; -import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterators; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieRestoreMetadata; @@ -63,12 +45,32 @@ import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; + public class TestIncrementalFSViewSync extends HoodieCommonTestHarness { private static final transient Logger log = LogManager.getLogger(TestIncrementalFSViewSync.class); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java index 0b84de1b5..b4fb33565 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java @@ -18,11 +18,12 @@ package org.apache.hudi.common.table.view; -import java.io.IOException; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.SyncableFileSystemView; +import java.io.IOException; + public class TestRocksDBBasedIncrementalFSViewSync extends TestIncrementalFSViewSync { @Override diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java index 9438d7e81..6cd102f0a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.table.view; -import java.io.IOException; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.SyncableFileSystemView; +import java.io.IOException; + public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemView { protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java index 4dbf810f2..fa2118b91 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java @@ -18,22 +18,6 @@ package org.apache.hudi.common.util; -import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; -import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; -import static org.apache.hudi.common.table.HoodieTimeline.DELTA_COMMIT_ACTION; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.model.FileSlice; @@ -46,8 +30,26 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.fs.Path; import org.junit.Assert; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; +import static org.apache.hudi.common.table.HoodieTimeline.DELTA_COMMIT_ACTION; + public class CompactionTestUtils { private static String TEST_WRITE_TOKEN = "1-0-1"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java index 725c091a5..3b07e1600 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java @@ -18,6 +18,19 @@ package org.apache.hudi.common.util; +import org.apache.hudi.avro.MercifulJsonConverter; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.DecoderFactory; + import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -33,17 +46,6 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.avro.io.DecoderFactory; -import org.apache.hudi.avro.MercifulJsonConverter; -import org.apache.hudi.common.model.HoodieAvroPayload; -import org.apache.hudi.common.model.HoodieKey; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; -import org.apache.hudi.exception.HoodieIOException; public class SchemaTestUtil { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java index 20515816c..0aacffd30 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java @@ -18,17 +18,19 @@ package org.apache.hudi.common.util; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + public class SpillableMapTestUtils { public static final String DUMMY_COMMIT_TIME = "DUMMY_COMMIT_TIME"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java index ad269b36f..475031943 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java @@ -18,22 +18,6 @@ package org.apache.hudi.common.util; -import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; -import static org.apache.hudi.common.util.CompactionTestUtils.createCompactionPlan; -import static org.apache.hudi.common.util.CompactionTestUtils.scheduleCompaction; -import static org.apache.hudi.common.util.CompactionTestUtils.setupAndValidateCompactionOperations; -import static org.apache.hudi.common.util.CompactionUtils.COMPACTION_METADATA_VERSION_1; -import static org.apache.hudi.common.util.CompactionUtils.LATEST_COMPACTION_METADATA_VERSION; - -import com.google.common.collect.ImmutableMap; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.HoodieCommonTestHarness; @@ -46,10 +30,28 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.CompactionTestUtils.TestHoodieDataFile; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.versioning.compaction.CompactionPlanMigrator; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.fs.Path; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.util.CompactionTestUtils.createCompactionPlan; +import static org.apache.hudi.common.util.CompactionTestUtils.scheduleCompaction; +import static org.apache.hudi.common.util.CompactionTestUtils.setupAndValidateCompactionOperations; +import static org.apache.hudi.common.util.CompactionUtils.COMPACTION_METADATA_VERSION_1; +import static org.apache.hudi.common.util.CompactionUtils.LATEST_COMPACTION_METADATA_VERSION; + public class TestCompactionUtils extends HoodieCommonTestHarness { private static String TEST_WRITE_TOKEN = "1-0-1"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java index f58603c0b..de4c74ae5 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java @@ -18,19 +18,21 @@ package org.apache.hudi.common.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import org.apache.hudi.common.minicluster.HdfsTestService; -import java.io.IOException; -import java.io.PrintStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hudi.common.minicluster.HdfsTestService; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.io.PrintStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + /** * Tests basic functionality of {@link DFSPropertiesConfiguration} */ diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java index 1b0eb6350..c381979d9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java @@ -18,8 +18,19 @@ package org.apache.hudi.common.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.hudi.common.HoodieCommonTestHarness; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.exception.HoodieException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; import java.io.IOException; import java.text.SimpleDateFormat; @@ -30,18 +41,9 @@ import java.util.List; import java.util.UUID; import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.HoodieCommonTestHarness; -import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.model.HoodieTestUtils; -import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.exception.HoodieException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class TestFSUtils extends HoodieCommonTestHarness { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java index 271213613..6ee9b63a9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java @@ -18,18 +18,19 @@ package org.apache.hudi.common.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import org.apache.hudi.common.HoodieCommonTestHarness; + +import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import org.apache.hudi.common.HoodieCommonTestHarness; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TestFileIOUtils extends HoodieCommonTestHarness { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java index 18a9c8d85..a34cdd7f3 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java @@ -18,12 +18,13 @@ package org.apache.hudi.common.util; -import java.util.Map; import org.apache.avro.Schema; import org.codehaus.jackson.JsonNode; import org.junit.Assert; import org.junit.Test; +import java.util.Map; + public class TestHoodieAvroUtils { private static String EXAMPLE_SCHEMA = "{\"type\": \"record\"," + "\"name\": \"testrec\"," + "\"fields\": [ " diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java index ad4c8eac7..3fde9055c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java @@ -18,10 +18,10 @@ package org.apache.hudi.common.util; -import static org.junit.Assert.assertTrue; - import org.junit.Test; +import static org.junit.Assert.assertTrue; + public class TestNumericUtils { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java index 807204346..93f415374 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java @@ -18,8 +18,21 @@ package org.apache.hudi.common.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.HoodieCommonTestHarness; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.avro.AvroSchemaConverter; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.junit.Before; +import org.junit.Test; import java.util.ArrayList; import java.util.Collections; @@ -27,20 +40,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.UUID; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.avro.HoodieAvroWriteSupport; -import org.apache.hudi.common.BloomFilter; -import org.apache.hudi.common.HoodieCommonTestHarness; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieTestUtils; -import org.apache.parquet.avro.AvroSchemaConverter; -import org.apache.parquet.hadoop.ParquetWriter; -import org.apache.parquet.hadoop.metadata.CompressionCodecName; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class TestParquetUtils extends HoodieCommonTestHarness { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java index e077f63e4..2e4c6db60 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; + import java.io.IOException; import java.io.Serializable; import java.util.HashMap; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java index 18695d2d5..5a708523f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java @@ -18,6 +18,14 @@ package org.apache.hudi.common.util; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.util.collection.Pair; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + import java.io.File; import java.io.Serializable; import java.util.Arrays; @@ -28,12 +36,6 @@ import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; -import org.apache.hudi.common.util.collection.Pair; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; public class TestRocksDBManager { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java index c79f524d6..65dac4db5 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java @@ -18,13 +18,14 @@ package org.apache.hudi.common.util; +import org.apache.avro.util.Utf8; +import org.junit.Assert; +import org.junit.Test; + import java.io.IOException; import java.util.Arrays; import java.util.LinkedList; import java.util.Objects; -import org.apache.avro.util.Utf8; -import org.junit.Assert; -import org.junit.Test; public class TestSerializationUtils { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java index 5b9d40069..671c60fb1 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java @@ -18,23 +18,6 @@ package org.apache.hudi.common.util.collection; -import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.AvroBinaryTestPayload; import org.apache.hudi.common.model.HoodieKey; @@ -47,10 +30,29 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SpillableMapTestUtils; import org.apache.hudi.common.util.SpillableMapUtils; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class TestDiskBasedMap extends HoodieCommonTestHarness { @Before diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java index c190f1288..c74e33a98 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java @@ -18,19 +18,6 @@ package org.apache.hudi.common.util.collection; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieKey; @@ -43,11 +30,26 @@ import org.apache.hudi.common.util.HoodieRecordSizeEstimator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SpillableMapTestUtils; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class TestExternalSpillableMap extends HoodieCommonTestHarness { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java index 172383109..e05a138e0 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java @@ -18,22 +18,24 @@ package org.apache.hudi.common.util.collection; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SpillableMapTestUtils; + +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + public class TestRocksDbBasedMap extends HoodieCommonTestHarness { @Before