lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-355] Refactor hudi-common based on new comment and code style rules (#1049)

Browse Source 
[HUDI-355] Refactor hudi-common based on new comment and code style rules
This commit is contained in:
vinoyang
2019-12-04 12:49:14 +08:00
committed by Balaji Varadarajan
parent 98ab33bb6e
commit 84602c8882
149 changed files with 637 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-common/src/test/java/org/apache/hudi/common/HoodieCommonTestHarness.java
View File

@@ -30,6 +30,9 @@ import org.junit.rules.TemporaryFolder;
import java.io.IOException;
/**
* The common hoodie test harness to provide the basic infrastructure.
*/
public class HoodieCommonTestHarness {
protected String basePath = null;

3
hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java
View File

@@ -22,6 +22,9 @@ import org.junit.Test;
import java.io.IOException;
/**
* Tests bloom filter {@link BloomFilter}.
*/
public class TestBloomFilter {
@Test

4
hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
View File

@@ -41,7 +41,7 @@ public class HdfsTestService {
private static final Logger logger = LogManager.getLogger(HdfsTestService.class);
/**
* Configuration settings
* Configuration settings.
*/
private Configuration hadoopConf;
private String workDir;
@@ -53,7 +53,7 @@ public class HdfsTestService {
private int datanodeHttpPort = 50075;
/**
* Embedded HDFS cluster
* Embedded HDFS cluster.
*/
private MiniDFSCluster miniDfsCluster;

3
hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java
View File

@@ -25,6 +25,9 @@ import org.apache.zookeeper.server.ZooKeeperServer;
import java.io.IOException;
/**
* A utility class about mini cluster.
*/
public class MiniClusterUtil {
private static MiniDFSCluster dfsCluster;

4
hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
View File

@@ -59,7 +59,7 @@ public class ZookeeperTestService {
private static final int CONNECTION_TIMEOUT = 30000;
/**
* Configuration settings
* Configuration settings.
*/
private Configuration hadoopConf;
private String workDir;
@@ -69,7 +69,7 @@ public class ZookeeperTestService {
private int tickTime = 0;
/**
* Embedded ZooKeeper cluster
* Embedded ZooKeeper cluster.
*/
private NIOServerCnxnFactory standaloneServerFactory;
private ZooKeeperServer zooKeeperServer;

3
hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java
View File

@@ -28,6 +28,9 @@ import org.apache.avro.generic.IndexedRecord;
import java.io.IOException;
/**
* The implementation of {@link HoodieRecordPayload} base on avro for testing.
*/
public class AvroBinaryTestPayload implements HoodieRecordPayload {
private final byte[] recordBytes;

3
hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
View File

@@ -80,6 +80,9 @@ import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
* A utility class for testing.
*/
public class HoodieTestUtils {
public static final String TEST_EXTENSION = ".test";

3
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java
View File

@@ -23,6 +23,9 @@ import org.junit.Test;
import java.util.List;
/**
* Tests hoodie commit metadata {@link HoodieCommitMetadata}.
*/
public class TestHoodieCommitMetadata {
@Test

3
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java
View File

@@ -31,6 +31,9 @@ import java.util.UUID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
/**
* Tests hoodie write stat {@link HoodieWriteStat}.
*/
public class TestHoodieWriteStat {
@Test

3
hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java
View File

@@ -24,6 +24,9 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
* Tests {@link StorageSchemes}.
*/
public class TestStorageSchemes {
@Test

3
hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
View File

@@ -42,6 +42,9 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* Tests hoodie table meta client {@link HoodieTableMetaClient}.
*/
public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
@Before

3
hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java
View File

@@ -76,6 +76,9 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
* Tests hoodie log format {@link HoodieLogFormat}.
*/
@SuppressWarnings("Duplicates")
@RunWith(Parameterized.class)
public class TestHoodieLogFormat extends HoodieCommonTestHarness {

2
hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java
View File

@@ -55,7 +55,7 @@ import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema;
/**
* This class is intentionally using a different way of setting up the MiniDFSCluster and not relying on
* {@link MiniClusterUtil} to reproduce append() issue : https://issues.apache.org/jira/browse/HDFS-6325 Reference :
* https://issues.apache.org/jira/secure/attachment/12645053/HDFS-6325.patch
* https://issues.apache.org/jira/secure/attachment/12645053/HDFS-6325.patch.
*/
public class TestHoodieLogFormatAppendFailure {

3
hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java
View File

@@ -28,6 +28,9 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* A mocked {@link HoodieActiveTimeline}.
*/
public class MockHoodieTimeline extends HoodieActiveTimeline {
public MockHoodieTimeline(Stream<String> completed, Stream<String> inflights) throws IOException {

3
hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
View File

@@ -37,6 +37,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* Tests {@link HoodieActiveTimeline}.
*/
public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
private HoodieActiveTimeline timeline;

5
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
View File

@@ -65,6 +65,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* Tests hoodie table file system view {@link HoodieTableFileSystemView}.
*/
@SuppressWarnings("ResultOfMethodCallIgnored")
public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
@@ -221,7 +224,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
}
/**
* Helper method to test Views in the presence of concurrent compaction
* Helper method to test Views in the presence of concurrent compaction.
*
* @param skipCreatingDataFile if set, first File Slice will not have data-file set. This would simulate inserts going
* directly to log files

31
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
View File

@@ -72,6 +72,9 @@ import java.util.stream.IntStream;
import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION;
/**
* Tests incremental file system view sync.
*/
public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
private static final transient Logger log = LogManager.getLogger(TestIncrementalFSViewSync.class);
@@ -189,7 +192,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Tests FS View incremental syncing behavior when multiple instants gets committed
* Tests FS View incremental syncing behavior when multiple instants gets committed.
*/
@Test
public void testMultipleTransitions() throws IOException {
@@ -291,7 +294,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
*********************************************************************************************************
*/
/**
* Helper to run one or more rounds of cleaning, incrementally syncing the view and then validate
* Helper to run one or more rounds of cleaning, incrementally syncing the view and then validate.
*/
private void testCleans(SyncableFileSystemView view, List<String> newCleanerInstants,
Map<String, List<String>> instantsToFiles, List<String> cleanedInstants) {
@@ -301,7 +304,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Simulates one of more cleaning, incrementally sync the view and validate the view
* Simulates one of more cleaning, incrementally sync the view and validate the view.
*
* @param view Hoodie View
* @param newCleanerInstants Cleaner Instants
@@ -352,7 +355,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Simulates one of more restores, incrementally sync the view and validate the view
* Simulates one of more restores, incrementally sync the view and validate the view.
*
* @param view Hoodie View
* @param newRestoreInstants Restore Instants
@@ -403,7 +406,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Simulate a Cleaner operation cleaning up an instant
* Simulate a Cleaner operation cleaning up an instant.
*
* @param view Hoodie View
* @param instant Instant to be cleaner
@@ -425,7 +428,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Simulate Restore of an instant in timeline and fsview
* Simulate Restore of an instant in timeline and fsview.
*
* @param view Hoodie View
* @param instant Instant to be rolled-back
@@ -466,7 +469,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Utility to delete a list of files and group the deleted files by partitions
* Utility to delete a list of files and group the deleted files by partitions.
*
* @param files List of files to be deleted
*/
@@ -489,7 +492,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Schedule a pending compaction and validate
* Schedule a pending compaction and validate.
*
* @param view Hoodie View
* @param instantTime COmpaction Instant Time
@@ -526,7 +529,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Unschedule a compaction instant and validate incremental fs view
* Unschedule a compaction instant and validate incremental fs view.
*
* @param view Hoodie View
* @param compactionInstantTime Compaction Instant to be removed
@@ -549,7 +552,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing.
*
* @param view Hoodie View
* @param instants Ingestion/Commit INstants
@@ -563,7 +566,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing.
*
* @param view Hoodie View
* @param instants Ingestion/Commit INstants
@@ -582,7 +585,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing.
*
* @param view Hoodie View
* @param instants Ingestion/Commit INstants
@@ -594,7 +597,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing
* Perform one or more rounds of ingestion/compaction and validate incremental timeline syncing.
*
* @param view Hoodie View
* @param instants Ingestion/Commit INstants
@@ -657,7 +660,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
}
/**
* Check for equality of views
* Check for equality of views.
*
* @param view1 View1
* @param view2 View2

3
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java
View File

@@ -24,6 +24,9 @@ import org.apache.hudi.common.table.SyncableFileSystemView;
import java.io.IOException;
/**
* Tests rocks db based incremental file system view sync {@link RocksDbBasedFileSystemView}.
*/
public class TestRocksDBBasedIncrementalFSViewSync extends TestIncrementalFSViewSync {
@Override

3
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
View File

@@ -23,6 +23,9 @@ import org.apache.hudi.common.table.SyncableFileSystemView;
import java.io.IOException;
/**
* Tests RocksDB based file system view {@link SyncableFileSystemView}.
*/
public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemView {
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException {

3
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java
View File

@@ -21,6 +21,9 @@ package org.apache.hudi.common.table.view;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.SyncableFileSystemView;
/**
* Tests spillable map based file system view {@link SyncableFileSystemView}.
*/
public class TestSpillableMapBasedFileSystemView extends TestHoodieTableFileSystemView {
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) {

3
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java
View File

@@ -22,6 +22,9 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.SyncableFileSystemView;
/**
* Tests spillable map based incremental fs view sync.
*/
public class TestSpillableMapBasedIncrementalFSViewSync extends TestIncrementalFSViewSync {
@Override

6
hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java
View File

@@ -50,6 +50,9 @@ import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PAT
import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION;
import static org.apache.hudi.common.table.HoodieTimeline.DELTA_COMMIT_ACTION;
/**
* The utility class to support testing compaction.
*/
public class CompactionTestUtils {
private static String TEST_WRITE_TOKEN = "1-0-1";
@@ -177,6 +180,9 @@ public class CompactionTestUtils {
CompactionUtils.LATEST_COMPACTION_METADATA_VERSION);
}
/**
* The hoodie data file for testing.
*/
public static class TestHoodieDataFile extends HoodieDataFile {
private final String path;

3
hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java
View File

@@ -47,6 +47,9 @@ import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* A utility class for testing schema.
*/
public class SchemaTestUtil {
public static Schema getSimpleSchema() throws IOException {

3
hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java
View File

@@ -31,6 +31,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Tests spillable map test utils.
*/
public class SpillableMapTestUtils {
public static final String DUMMY_COMMIT_TIME = "DUMMY_COMMIT_TIME";

9
hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
View File

@@ -52,6 +52,9 @@ import static org.apache.hudi.common.util.CompactionTestUtils.setupAndValidateCo
import static org.apache.hudi.common.util.CompactionUtils.COMPACTION_METADATA_VERSION_1;
import static org.apache.hudi.common.util.CompactionUtils.LATEST_COMPACTION_METADATA_VERSION;
/**
* The utility class for testing compaction.
*/
public class TestCompactionUtils extends HoodieCommonTestHarness {
private static String TEST_WRITE_TOKEN = "1-0-1";
@@ -120,7 +123,7 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
}
/**
* Generate input for compaction plan tests
* Generate input for compaction plan tests.
*/
private Pair<List<Pair<String, FileSlice>>, HoodieCompactionPlan> buildCompactionPlan() {
Path fullPartitionPath = new Path(new Path(metaClient.getBasePath()), DEFAULT_PARTITION_PATHS[0]);
@@ -215,7 +218,7 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
}
/**
* Validates if generated compaction plan matches with input file-slices
* Validates if generated compaction plan matches with input file-slices.
*
* @param input File Slices with partition-path
* @param plan Compaction Plan
@@ -227,7 +230,7 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
}
/**
* Validates if generated compaction operation matches with input file slice and partition path
* Validates if generated compaction operation matches with input file slice and partition path.
*
* @param slice File Slice
* @param op HoodieCompactionOperation

2
hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
View File

@@ -34,7 +34,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
* Tests basic functionality of {@link DFSPropertiesConfiguration}
* Tests basic functionality of {@link DFSPropertiesConfiguration}.
*/
public class TestDFSPropertiesConfiguration {

3
hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java
View File

@@ -45,6 +45,9 @@ import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Tests file system utils.
*/
public class TestFSUtils extends HoodieCommonTestHarness {
private static String TEST_WRITE_TOKEN = "1-0-1";

3
hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java
View File

@@ -32,6 +32,9 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
* Tests file I/O utils.
*/
public class TestFileIOUtils extends HoodieCommonTestHarness {
@Test

3
hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java
View File

@@ -25,6 +25,9 @@ import org.junit.Test;
import java.util.Map;
/**
* Tests hoodie avro utilities.
*/
public class TestHoodieAvroUtils {
private static String EXAMPLE_SCHEMA = "{\"type\": \"record\"," + "\"name\": \"testrec\"," + "\"fields\": [ "

3
hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java
View File

@@ -22,6 +22,9 @@ import org.junit.Test;
import static org.junit.Assert.assertTrue;
/**
* Tests numeric utils.
*/
public class TestNumericUtils {
@Test

3
hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
View File

@@ -44,6 +44,9 @@ import java.util.UUID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Tests parquet utils.
*/
public class TestParquetUtils extends HoodieCommonTestHarness {
@Before

3
hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java
View File

@@ -28,6 +28,9 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
/**
* A test record entity.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@SuppressWarnings({"unused", "FieldCanBeLocal", "MismatchedQueryAndUpdateOfCollection"})
public class TestRecord implements Serializable {

6
hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java
View File

@@ -37,6 +37,9 @@ import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* Tests RocksDB manager {@link RocksDBDAO}.
*/
public class TestRocksDBManager {
private static RocksDBDAO dbManager;
@@ -133,6 +136,9 @@ public class TestRocksDBManager {
Assert.assertFalse(new File(rocksDBBasePath).exists());
}
/**
* A payload definition for {@link TestRocksDBManager}.
*/
public static class Payload implements Serializable {
private final String prefix;

3
hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java
View File

@@ -27,6 +27,9 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.Objects;
/**
* Tests serialization utils.
*/
public class TestSerializationUtils {
@Test

3
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java
View File

@@ -53,6 +53,9 @@ import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Tests dis based map {@link DiskBasedMap}.
*/
public class TestDiskBasedMap extends HoodieCommonTestHarness {
@Before

3
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
View File

@@ -50,6 +50,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* Tests external spillable map {@link ExternalSpillableMap}.
*/
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class TestExternalSpillableMap extends HoodieCommonTestHarness {

3
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java
View File

@@ -36,6 +36,9 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
* Tests RocksDB based map {@link RocksDBBasedMap}.
*/
public class TestRocksDbBasedMap extends HoodieCommonTestHarness {
@Before