From 03f136361a5fed594855992ab10bee8bb5060c5b Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Wed, 27 May 2020 01:28:17 -0700 Subject: [PATCH] [HUDI-811] Restructure test packages in hudi-common (#1644) * [HUDI-811] Restructure test packages in hudi-common --- .../commands/TestHoodieLogFileCommand.java | 20 +- .../cli/commands/TestRollbacksCommand.java | 2 +- .../hudi/cli/commands/TestStatsCommand.java | 2 +- .../HoodieTestCommitMetadataGenerator.java | 2 +- .../hudi/cli/integ/ITTestRepairsCommand.java | 13 +- .../hudi/client/TestClientRollback.java | 2 +- .../client/TestCompactionAdminClient.java | 4 +- .../TestHoodieClientOnCopyOnWriteStorage.java | 2 +- .../org/apache/hudi/client/TestMultiFS.java | 2 +- .../client/TestUpdateSchemaEvolution.java | 2 +- .../hudi/common/HoodieClientTestHarness.java | 4 +- .../hudi/common/HoodieClientTestUtils.java | 2 +- .../common/HoodieMergeOnReadTestUtils.java | 4 +- .../hudi/io/TestHoodieCommitArchiveLog.java | 2 +- .../io/TestHoodieKeyLocationFetchHandle.java | 2 +- .../org/apache/hudi/table/TestCleaner.java | 4 +- .../hudi/table/TestHoodieRecordSizing.java | 2 +- .../hudi/table/TestMergeOnReadTable.java | 2 +- .../commit/TestCopyOnWriteActionExecutor.java | 2 +- .../action/compact/TestAsyncCompaction.java | 2 +- .../action/compact/TestHoodieCompactor.java | 2 +- .../common/table/log/HoodieLogFileReader.java | 8 +- .../apache/hudi/common/fs/TestFSUtils.java | 2 +- .../{storage => fs}/TestStorageSchemes.java | 4 +- .../fs/inline/TestInLineFileSystem.java | 5 +- ...=> TestInLineFileSystemHFileInLining.java} | 14 +- .../fs/inline/TestInMemoryFileSystem.java | 4 +- .../TestHoodieLogFormat.java | 13 +- .../TestHoodieLogFormatAppendFailure.java | 9 +- .../model/TestHoodieCommitMetadata.java | 1 + .../hudi/common/model/TestHoodieRecord.java | 3 +- .../table/TestHoodieTableMetaClient.java | 3 +- .../timeline/TestHoodieActiveTimeline.java | 3 +- .../{ => timeline}/TestTimelineLayout.java | 5 +- .../view/TestPriorityBasedFileSystemView.java | 2 +- .../view/TestRocksDbBasedFileSystemView.java | 3 +- .../TestSpillableMapBasedFileSystemView.java | 3 +- ...pillableMapBasedIncrementalFSViewSync.java | 2 +- .../AvroBinaryTestPayload.java | 3 +- .../CompactionTestUtils.java | 8 +- .../FileSystemTestUtils.java | 12 +- .../testutils/HoodieCommonTestHarness.java | 1 - .../{model => testutils}/HoodieTestUtils.java | 10 +- .../MockHoodieTimeline.java | 6 +- .../SampleTestRecord.java} | 8 +- .../{util => testutils}/SchemaTestUtil.java | 7 +- .../SpillableMapTestUtils.java | 3 +- .../minicluster/HdfsTestService.java | 4 +- .../minicluster/MiniClusterUtil.java | 2 +- .../minicluster/ZookeeperTestService.java | 2 +- .../hudi/common/util/TestCompactionUtils.java | 12 +- .../util/TestDFSPropertiesConfiguration.java | 2 +- .../hudi/common/util/TestParquetUtils.java | 2 +- .../util/collection/TestDiskBasedMap.java | 8 +- .../collection/TestExternalSpillableMap.java | 4 +- ...ocksDBManager.java => TestRocksDBDAO.java} | 4 +- .../util/collection/TestRocksDbBasedMap.java | 4 +- .../hadoop/TestHoodieParquetInputFormat.java | 2 +- .../hadoop/TestHoodieROTablePathFilter.java | 2 +- .../hudi/hadoop/TestInputPathHandler.java | 2 +- .../TestHoodieCombineHiveInputFormat.java | 9 +- .../TestHoodieRealtimeRecordReader.java | 4 +- .../hadoop/testutils/InputFormatTestUtil.java | 4 +- .../apache/hudi/hive/TestHiveSyncTool.java | 226 +++++++++--------- .../hudi/hive/testutils/HiveTestService.java | 2 +- .../{TestUtil.java => HiveTestUtil.java} | 8 +- .../apache/hudi/TestDataSourceDefaults.scala | 3 +- .../TestKafkaConnectHdfsProvider.java | 2 +- .../functional/TestHDFSParquetImporter.java | 4 +- .../functional/TestHoodieSnapshotCopier.java | 2 +- .../inline/fs/TestParquetInLining.java | 8 +- .../TestTimestampBasedKeyGenerator.java | 2 +- .../testutils/UtilitiesTestBase.java | 4 +- 73 files changed, 288 insertions(+), 261 deletions(-) rename hudi-common/src/test/java/org/apache/hudi/common/{storage => fs}/TestStorageSchemes.java (95%) rename hudi-common/src/test/java/org/apache/hudi/common/fs/inline/{TestHFileInLining.java => TestInLineFileSystemHFileInLining.java} (95%) rename hudi-common/src/test/java/org/apache/hudi/common/{table/log => functional}/TestHoodieLogFormat.java (99%) rename hudi-common/src/test/java/org/apache/hudi/common/{table/log => functional}/TestHoodieLogFormatAppendFailure.java (95%) rename hudi-common/src/test/java/org/apache/hudi/common/table/{ => timeline}/TestTimelineLayout.java (95%) rename hudi-common/src/test/java/org/apache/hudi/common/{model => testutils}/AvroBinaryTestPayload.java (95%) rename hudi-common/src/test/java/org/apache/hudi/common/{util => testutils}/CompactionTestUtils.java (97%) rename hudi-common/src/test/java/org/apache/hudi/common/{fs/inline => testutils}/FileSystemTestUtils.java (87%) rename hudi-common/src/test/java/org/apache/hudi/common/{model => testutils}/HoodieTestUtils.java (97%) rename hudi-common/src/test/java/org/apache/hudi/common/{table/timeline => testutils}/MockHoodieTimeline.java (86%) rename hudi-common/src/test/java/org/apache/hudi/common/{util/TestRecord.java => testutils/SampleTestRecord.java} (94%) rename hudi-common/src/test/java/org/apache/hudi/common/{util => testutils}/SchemaTestUtil.java (97%) rename hudi-common/src/test/java/org/apache/hudi/common/{util => testutils}/SpillableMapTestUtils.java (96%) rename hudi-common/src/test/java/org/apache/hudi/common/{ => testutils}/minicluster/HdfsTestService.java (97%) rename hudi-common/src/test/java/org/apache/hudi/common/{ => testutils}/minicluster/MiniClusterUtil.java (97%) rename hudi-common/src/test/java/org/apache/hudi/common/{ => testutils}/minicluster/ZookeeperTestService.java (99%) rename hudi-common/src/test/java/org/apache/hudi/common/util/collection/{TestRocksDBManager.java => TestRocksDBDAO.java} (99%) rename hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/{hive => functional}/TestHoodieCombineHiveInputFormat.java (95%) rename hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/{TestUtil.java => HiveTestUtil.java} (98%) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java index b0d250419..e490a77e9 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java @@ -18,16 +18,11 @@ package org.apache.hudi.cli.commands; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; -import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.TableHeader; import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.model.HoodieLogFile; @@ -39,13 +34,18 @@ import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieMemoryConfig; -import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; import java.io.File; @@ -53,15 +53,15 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Arrays; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; +import static org.apache.hudi.common.testutils.SchemaTestUtil.getSimpleSchema; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java index 5a82d778f..d8903aee5 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java @@ -27,11 +27,11 @@ import org.apache.hudi.cli.TableHeader; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java index 85fbc0ab0..3305aa0b4 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java @@ -29,8 +29,8 @@ import org.apache.hudi.cli.TableHeader; import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java b/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java index 2c4719cb2..c239e0d8c 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java @@ -21,10 +21,10 @@ package org.apache.hudi.cli.common; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieCommitMetadata; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java index 4f48bc34f..6af25b6d1 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java @@ -18,9 +18,6 @@ package org.apache.hudi.cli.integ; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; @@ -36,11 +33,14 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; -import org.apache.hudi.common.util.SchemaTestUtil; -import org.apache.spark.sql.Dataset; +import org.apache.hudi.common.testutils.SchemaTestUtil; -import org.junit.jupiter.api.Test; +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.spark.sql.Dataset; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; import java.io.File; @@ -54,7 +54,6 @@ import java.util.UUID; import java.util.stream.Collectors; import static org.apache.spark.sql.functions.lit; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java b/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java index bdfd380bb..b2976b572 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java @@ -23,10 +23,10 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java b/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java index ae7438fa1..a9d45eafb 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java @@ -23,10 +23,10 @@ import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; -import org.apache.hudi.common.util.CompactionTestUtils; +import org.apache.hudi.common.testutils.CompactionTestUtils; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CompactionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java index b50c4cbae..66295111d 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java @@ -29,13 +29,13 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRollingStat; import org.apache.hudi.common.model.HoodieRollingStatMetadata; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ParquetUtils; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java index 54e5e8b0a..ad8b87f2c 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java @@ -25,10 +25,10 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java index 2e5a18a3b..303be2ac3 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java @@ -24,8 +24,8 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.config.HoodieWriteConfig; diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestHarness.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestHarness.java index 2a24c210a..2b336695b 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestHarness.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestHarness.java @@ -20,10 +20,10 @@ package org.apache.hudi.common; import org.apache.hudi.client.SparkTaskContextSupplier; import org.apache.hudi.client.TestHoodieClientBase; import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.minicluster.HdfsTestService; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.minicluster.HdfsTestService; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java index 0652091f1..4882469e6 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java @@ -31,12 +31,12 @@ import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.io.storage.HoodieParquetConfig; diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java index 1a65a46a2..e54b53b8e 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java @@ -19,7 +19,7 @@ package org.apache.hudi.common; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.hadoop.HoodieParquetInputFormat; import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat; @@ -125,4 +125,4 @@ public class HoodieMergeOnReadTestUtils { jobConf.set("mapreduce.input.fileinputformat.inputdir", inputPath); jobConf.set("map.input.dir", inputPath); } -} \ No newline at end of file +} diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java index bfd594670..09f4616a4 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java @@ -21,7 +21,6 @@ package org.apache.hudi.io; import org.apache.hudi.common.HoodieClientTestHarness; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieCommitMetadata; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; @@ -29,6 +28,7 @@ import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTimelineArchiveLog; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java index 50e212733..c4262921f 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java @@ -28,10 +28,10 @@ import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java index 844551846..bc5a43b54 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -36,7 +36,6 @@ import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; @@ -46,6 +45,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.table.timeline.versioning.clean.CleanMetadataMigrator; import org.apache.hudi.common.table.view.TableFileSystemView; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CleanerUtils; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.CompactionUtils; @@ -80,7 +80,7 @@ import java.util.stream.Stream; import scala.Tuple3; -import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_PARTITION_PATHS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java index 1ce87f074..17b005091 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieRecordSizing.java @@ -33,7 +33,7 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import static org.apache.hudi.common.model.HoodieTestUtils.generateFakeHoodieWriteStat; +import static org.apache.hudi.common.testutils.HoodieTestUtils.generateFakeHoodieWriteStat; import static org.apache.hudi.table.HoodieCopyOnWriteTable.averageBytesPerRecord; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java index bcdffba87..99773b92e 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java @@ -35,7 +35,6 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRollingStat; import org.apache.hudi.common.model.HoodieRollingStatMetadata; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -45,6 +44,7 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.table.view.SyncableFileSystemView; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; import org.apache.hudi.common.table.view.TableFileSystemView.SliceView; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java index d39df241d..3fc260100 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java @@ -29,8 +29,8 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.common.util.collection.Pair; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java index 88fdae80c..9b5312c70 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java @@ -31,7 +31,6 @@ import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; @@ -39,6 +38,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CompactionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java index 9aec8adbc..1c4d26e01 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java @@ -27,12 +27,12 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java index da3a37572..1f349adf6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java @@ -52,7 +52,7 @@ import java.util.Objects; * Scans a log file and provides block level iterator on the log file Loads the entire block contents in memory Can emit * either a DataBlock, CommandBlock, DeleteBlock or CorruptBlock (if one is found). */ -class HoodieLogFileReader implements HoodieLogFormat.Reader { +public class HoodieLogFileReader implements HoodieLogFormat.Reader { public static final int DEFAULT_BUFFER_SIZE = 16 * 1024 * 1024; // 16 MB private static final Logger LOG = LogManager.getLogger(HoodieLogFileReader.class); @@ -67,7 +67,7 @@ class HoodieLogFileReader implements HoodieLogFormat.Reader { private boolean reverseReader; private boolean closed = false; - HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize, + public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize, boolean readBlockLazily, boolean reverseReader) throws IOException { FSDataInputStream fsDataInputStream = fs.open(logFile.getPath(), bufferSize); if (fsDataInputStream.getWrappedStream() instanceof FSInputStream) { @@ -89,12 +89,12 @@ class HoodieLogFileReader implements HoodieLogFormat.Reader { addShutDownHook(); } - HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, boolean readBlockLazily, + public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, boolean readBlockLazily, boolean reverseReader) throws IOException { this(fs, logFile, readerSchema, DEFAULT_BUFFER_SIZE, readBlockLazily, reverseReader); } - HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema) throws IOException { + public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema) throws IOException { this(fs, logFile, readerSchema, DEFAULT_BUFFER_SIZE, false, false); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index b6a86a1e2..0e35df5f2 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -19,11 +19,11 @@ package org.apache.hudi.common.fs; import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.exception.HoodieException; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java similarity index 95% rename from hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java rename to hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java index f3b6c9f91..4cf826077 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java @@ -16,9 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common.storage; - -import org.apache.hudi.common.fs.StorageSchemes; +package org.apache.hudi.common.fs; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java index d2adbb360..32394cb19 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hudi.common.fs.inline; +import org.apache.hudi.common.testutils.FileSystemTestUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hadoop.conf.Configuration; @@ -38,8 +39,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.RANDOM; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.getRandomOuterFSPath; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.RANDOM; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.getRandomOuterFSPath; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestHFileInLining.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLining.java similarity index 95% rename from hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestHFileInLining.java rename to hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLining.java index 96d939ad0..4122d500b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestHFileInLining.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLining.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.fs.inline; +import org.apache.hudi.common.testutils.FileSystemTestUtils; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -40,10 +42,10 @@ import java.util.HashSet; import java.util.Set; import java.util.UUID; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.FILE_SCHEME; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.RANDOM; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.getPhantomFile; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.getRandomOuterInMemPath; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.FILE_SCHEME; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.RANDOM; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.getPhantomFile; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.getRandomOuterInMemPath; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -51,7 +53,7 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; /** * Tests {@link InLineFileSystem} to inline HFile. */ -public class TestHFileInLining { +public class TestInLineFileSystemHFileInLining { private final Configuration inMemoryConf; private final Configuration inlineConf; @@ -60,7 +62,7 @@ public class TestHFileInLining { private int maxRows = 100 + RANDOM.nextInt(1000); private Path generatedPath; - public TestHFileInLining() { + public TestInLineFileSystemHFileInLining() { inMemoryConf = new Configuration(); inMemoryConf.set("fs." + InMemoryFileSystem.SCHEME + ".impl", InMemoryFileSystem.class.getName()); inlineConf = new Configuration(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java index 260607408..417222562 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java @@ -26,8 +26,8 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URI; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.RANDOM; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.getRandomOuterInMemPath; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.RANDOM; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.getRandomOuterInMemPath; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java similarity index 99% rename from hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java rename to hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java index 68f755827..e9f0ef7b1 100755 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java @@ -16,19 +16,20 @@ * limitations under the License. */ -package org.apache.hudi.common.table.log; +package org.apache.hudi.common.functional; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.minicluster.MiniClusterUtil; import org.apache.hudi.common.model.HoodieArchivedLogFile; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.log.HoodieLogFileReader; +import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; @@ -36,7 +37,9 @@ import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; -import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.common.testutils.minicluster.MiniClusterUtil; import org.apache.hudi.exception.CorruptedLogFileException; import org.apache.avro.Schema; @@ -67,7 +70,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; +import static org.apache.hudi.common.testutils.SchemaTestUtil.getSimpleSchema; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java similarity index 95% rename from hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java rename to hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java index 58396462b..201ed4fc6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java @@ -16,14 +16,15 @@ * limitations under the License. */ -package org.apache.hudi.common.table.log; +package org.apache.hudi.common.functional; -import org.apache.hudi.common.minicluster.MiniClusterUtil; import org.apache.hudi.common.model.HoodieArchivedLogFile; +import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; -import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.common.testutils.minicluster.MiniClusterUtil; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.conf.Configuration; @@ -50,7 +51,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeoutException; -import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; +import static org.apache.hudi.common.testutils.SchemaTestUtil.getSimpleSchema; import static org.junit.jupiter.api.Assertions.assertNotEquals; /** diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java index 844359f3e..0eaaff126 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java @@ -18,6 +18,7 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.FileIOUtils; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java index 6fbe9a545..e31286d10 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java @@ -18,8 +18,9 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.testutils.AvroBinaryTestPayload; +import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java index 88d356715..121e173c6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java @@ -18,11 +18,11 @@ package org.apache.hudi.common.table; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; import org.junit.jupiter.api.BeforeEach; @@ -37,7 +37,6 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Tests hoodie table meta client {@link HoodieTableMetaClient}. */ diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java index 0915cf25c..4fc9eaca6 100755 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java @@ -18,11 +18,12 @@ package org.apache.hudi.common.table.timeline; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.MockHoodieTimeline; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineLayout.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestTimelineLayout.java similarity index 95% rename from hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineLayout.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestTimelineLayout.java index 038692267..41cf4abf7 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineLayout.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestTimelineLayout.java @@ -16,12 +16,9 @@ * limitations under the License. */ -package org.apache.hudi.common.table; +package org.apache.hudi.common.table.timeline; -import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; -import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.common.table.timeline.TimelineLayout; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java index e59860e30..bae749611 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java @@ -24,7 +24,7 @@ import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.common.table.timeline.MockHoodieTimeline; +import org.apache.hudi.common.testutils.MockHoodieTimeline; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ImmutablePair; import org.apache.hudi.common.util.collection.Pair; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java index b09b007bf..7793178d1 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java @@ -24,10 +24,11 @@ import java.io.IOException; import java.nio.file.Files; /** - * Tests RocksDB based file system view {@link SyncableFileSystemView}. + * Tests RocksDB based file system view {@link RocksDbBasedFileSystemView}. */ public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemView { + @Override protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException { String subdirPath = Files.createTempDirectory(tempDir, null).toAbsolutePath().toString(); return new RocksDbBasedFileSystemView(metaClient, timeline, diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java index 35324e7c8..7f2e0dc29 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java @@ -21,10 +21,11 @@ package org.apache.hudi.common.table.view; import org.apache.hudi.common.table.timeline.HoodieTimeline; /** - * Tests spillable map based file system view {@link SyncableFileSystemView}. + * Tests spillable map based file system view {@link SpillableMapBasedFileSystemView}. */ public class TestSpillableMapBasedFileSystemView extends TestHoodieTableFileSystemView { + @Override protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) { return new SpillableMapBasedFileSystemView(metaClient, timeline, FileSystemViewStorageConfig.newBuilder() // pure disk base View diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java index 8384d42fe..d3478ce14 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java @@ -22,7 +22,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieTimeline; /** - * Tests spillable map based incremental fs view sync. + * Tests spillable map based incremental fs view sync {@link SpillableMapBasedFileSystemView}. */ public class TestSpillableMapBasedIncrementalFSViewSync extends TestIncrementalFSViewSync { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/AvroBinaryTestPayload.java similarity index 95% rename from hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/AvroBinaryTestPayload.java index 440d92bd5..7ea3038c3 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/AvroBinaryTestPayload.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package org.apache.hudi.common.model; +package org.apache.hudi.common.testutils; import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java similarity index 97% rename from hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java index baf34e0b2..7d9110b2e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CompactionTestUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common.util; +package org.apache.hudi.common.testutils; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; @@ -25,11 +25,13 @@ import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; +import org.apache.hudi.common.util.CollectionUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; @@ -45,9 +47,9 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMPACTION_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION; +import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_PARTITION_PATHS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/FileSystemTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java similarity index 87% rename from hudi-common/src/test/java/org/apache/hudi/common/fs/inline/FileSystemTestUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java index c64b04a45..8432b26f8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/FileSystemTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java @@ -16,7 +16,11 @@ * limitations under the License. */ -package org.apache.hudi.common.fs.inline; +package org.apache.hudi.common.testutils; + +import org.apache.hudi.common.fs.inline.InLineFSUtils; +import org.apache.hudi.common.fs.inline.InLineFileSystem; +import org.apache.hudi.common.fs.inline.InMemoryFileSystem; import org.apache.hadoop.fs.Path; @@ -34,7 +38,7 @@ public class FileSystemTestUtils { public static final String FORWARD_SLASH = "/"; public static final String FILE_SCHEME = "file"; public static final String COLON = ":"; - static final Random RANDOM = new Random(); + public static final Random RANDOM = new Random(); public static Path getRandomOuterInMemPath() { String randomFileName = UUID.randomUUID().toString(); @@ -42,7 +46,7 @@ public class FileSystemTestUtils { return new Path(InMemoryFileSystem.SCHEME + fileSuffix); } - static Path getRandomOuterFSPath() { + public static Path getRandomOuterFSPath() { String randomFileName = UUID.randomUUID().toString(); String fileSuffix = COLON + FORWARD_SLASH + TEMP + FORWARD_SLASH + randomFileName; return new Path(FILE_SCHEME + fileSuffix); @@ -63,4 +67,4 @@ public class FileSystemTestUtils { throw new IOException(message); } } -} \ No newline at end of file +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java index 4315b50ab..8e1d080bc 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java @@ -19,7 +19,6 @@ package org.apache.hudi.common.testutils; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java similarity index 97% rename from hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java index a718cef82..18dce0329 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common.model; +package org.apache.hudi.common.testutils; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.model.HoodieActionInstant; @@ -25,6 +25,14 @@ import org.apache.hudi.avro.model.HoodieCleanerPlan; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.HoodieCleanStat; import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/MockHoodieTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/MockHoodieTimeline.java similarity index 86% rename from hudi-common/src/test/java/org/apache/hudi/common/table/timeline/MockHoodieTimeline.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/MockHoodieTimeline.java index 8e812830a..5da6b325f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/MockHoodieTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/MockHoodieTimeline.java @@ -16,7 +16,11 @@ * limitations under the License. */ -package org.apache.hudi.common.table.timeline; +package org.apache.hudi.common.testutils; + +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; import java.util.Comparator; import java.util.stream.Collectors; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SampleTestRecord.java similarity index 94% rename from hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/SampleTestRecord.java index 2afc25955..f8e6252a2 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SampleTestRecord.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common.util; +package org.apache.hudi.common.testutils; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -29,11 +29,11 @@ import java.util.HashMap; import java.util.Map; /** - * A test record entity. + * A sample record entity for tests. */ @JsonIgnoreProperties(ignoreUnknown = true) @SuppressWarnings({"unused", "FieldCanBeLocal", "MismatchedQueryAndUpdateOfCollection"}) -public class TestRecord implements Serializable { +public class SampleTestRecord implements Serializable { class TestMapItemRecord implements Serializable { @@ -74,7 +74,7 @@ public class TestRecord implements Serializable { private TestNestedRecord testNestedRecord; private String[] stringArray; - public TestRecord(String instantTime, int recordNumber, String fileId) { + public SampleTestRecord(String instantTime, int recordNumber, String fileId) { this._hoodie_commit_time = instantTime; this._hoodie_record_key = "key" + recordNumber; this._hoodie_partition_path = instantTime; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java similarity index 97% rename from hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java index 04546f368..140d95a88 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common.util; +package org.apache.hudi.common.testutils; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.MercifulJsonConverter; @@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; import org.apache.avro.Schema; @@ -86,7 +87,7 @@ public class SchemaTestUtil { } } - static Path uriToPath(URI uri) throws IOException { + public static Path uriToPath(URI uri) throws IOException { final Map env = new HashMap<>(); final String[] array = uri.toString().split("!"); FileSystem fs; @@ -176,7 +177,7 @@ public class SchemaTestUtil { public static GenericRecord generateAvroRecordFromJson(Schema schema, int recordNumber, String instantTime, String fileId) throws IOException { - TestRecord record = new TestRecord(instantTime, recordNumber, fileId); + SampleTestRecord record = new SampleTestRecord(instantTime, recordNumber, fileId); MercifulJsonConverter converter = new MercifulJsonConverter(); return converter.convert(record.toJsonString(), schema); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SpillableMapTestUtils.java similarity index 96% rename from hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/SpillableMapTestUtils.java index 1d411d87e..89155904e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SpillableMapTestUtils.java @@ -16,13 +16,14 @@ * limitations under the License. */ -package org.apache.hudi.common.util; +package org.apache.hudi.common.testutils; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java similarity index 97% rename from hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java index 00e6e3c7b..a50e69d6f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.hudi.common.minicluster; +package org.apache.hudi.common.testutils.minicluster; -import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.testutils.NetworkTestUtils; import org.apache.hudi.common.util.FileIOUtils; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/MiniClusterUtil.java similarity index 97% rename from hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/MiniClusterUtil.java index b1d85dd42..ecc00e5f6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/MiniClusterUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common.minicluster; +package org.apache.hudi.common.testutils.minicluster; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java similarity index 99% rename from hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java index 5c837dd3c..c27877008 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common.minicluster; +package org.apache.hudi.common.testutils.minicluster; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java index 269301508..5e722fdaa 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java @@ -28,8 +28,8 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.compaction.CompactionPlanMigrator; +import org.apache.hudi.common.testutils.CompactionTestUtils.TestHoodieBaseFile; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; -import org.apache.hudi.common.util.CompactionTestUtils.TestHoodieBaseFile; import org.apache.hudi.common.util.collection.Pair; import org.apache.hadoop.fs.Path; @@ -45,10 +45,10 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; -import static org.apache.hudi.common.util.CompactionTestUtils.createCompactionPlan; -import static org.apache.hudi.common.util.CompactionTestUtils.scheduleCompaction; -import static org.apache.hudi.common.util.CompactionTestUtils.setupAndValidateCompactionOperations; +import static org.apache.hudi.common.testutils.CompactionTestUtils.createCompactionPlan; +import static org.apache.hudi.common.testutils.CompactionTestUtils.scheduleCompaction; +import static org.apache.hudi.common.testutils.CompactionTestUtils.setupAndValidateCompactionOperations; +import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_PARTITION_PATHS; import static org.apache.hudi.common.util.CompactionUtils.COMPACTION_METADATA_VERSION_1; import static org.apache.hudi.common.util.CompactionUtils.LATEST_COMPACTION_METADATA_VERSION; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -56,7 +56,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * The utility class for testing compaction. + * Tests for {@link CompactionUtils}. */ public class TestCompactionUtils extends HoodieCommonTestHarness { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java index 792eded41..ddce3216b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java @@ -20,7 +20,7 @@ package org.apache.hudi.common.util; import org.apache.hudi.common.config.DFSPropertiesConfiguration; import org.apache.hudi.common.config.TypedProperties; -import org.apache.hudi.common.minicluster.HdfsTestService; +import org.apache.hudi.common.testutils.minicluster.HdfsTestService; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java index 15b160292..9496f0195 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java @@ -25,8 +25,8 @@ import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.bloom.BloomFilterTypeCode; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java index 32119cc1e..e3cc88656 100755 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java @@ -19,17 +19,17 @@ package org.apache.hudi.common.util.collection; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.model.AvroBinaryTestPayload; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.testutils.AvroBinaryTestPayload; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.common.testutils.SpillableMapTestUtils; import org.apache.hudi.common.util.HoodieRecordSizeEstimator; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SchemaTestUtil; -import org.apache.hudi.common.util.SpillableMapTestUtils; import org.apache.hudi.common.util.SpillableMapUtils; import org.apache.avro.Schema; @@ -52,7 +52,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; +import static org.apache.hudi.common.testutils.SchemaTestUtil.getSimpleSchema; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java index 23a14d906..b240c8dd1 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java @@ -25,11 +25,11 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.common.testutils.SpillableMapTestUtils; import org.apache.hudi.common.util.DefaultSizeEstimator; import org.apache.hudi.common.util.HoodieRecordSizeEstimator; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SchemaTestUtil; -import org.apache.hudi.common.util.SpillableMapTestUtils; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBManager.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBDAO.java similarity index 99% rename from hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBManager.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBDAO.java index 2a5691e02..0c2cf3ffe 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBManager.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBDAO.java @@ -45,7 +45,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests RocksDB manager {@link RocksDBDAO}. */ -public class TestRocksDBManager { +public class TestRocksDBDAO { private RocksDBDAO dbManager; @@ -269,7 +269,7 @@ public class TestRocksDBManager { } /** - * A payload definition for {@link TestRocksDBManager}. + * A payload definition for {@link TestRocksDBDAO}. */ public static class Payload implements Serializable { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java index 86e1f0c21..1111d10e4 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java @@ -21,8 +21,8 @@ package org.apache.hudi.common.util.collection; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; -import org.apache.hudi.common.util.SchemaTestUtil; -import org.apache.hudi.common.util.SpillableMapTestUtils; +import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.common.testutils.SpillableMapTestUtils; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieParquetInputFormat.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieParquetInputFormat.java index 9ff624180..51a6524db 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieParquetInputFormat.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieParquetInputFormat.java @@ -21,13 +21,13 @@ package org.apache.hudi.hadoop; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieCommitMetadata; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.hadoop.testutils.InputFormatTestUtil; import org.apache.avro.Schema; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java index 040a89b46..18e9afd73 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java @@ -18,9 +18,9 @@ package org.apache.hudi.hadoop; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java index d5acf6864..3a8b19744 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java @@ -18,11 +18,11 @@ package org.apache.hudi.hadoop; -import org.apache.hudi.common.minicluster.HdfsTestService; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.minicluster.HdfsTestService; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/hive/TestHoodieCombineHiveInputFormat.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/functional/TestHoodieCombineHiveInputFormat.java similarity index 95% rename from hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/hive/TestHoodieCombineHiveInputFormat.java rename to hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/functional/TestHoodieCombineHiveInputFormat.java index 93d2f3edc..5150d6abc 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/hive/TestHoodieCombineHiveInputFormat.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/functional/TestHoodieCombineHiveInputFormat.java @@ -16,15 +16,16 @@ * limitations under the License. */ -package org.apache.hudi.hadoop.hive; +package org.apache.hudi.hadoop.functional; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.minicluster.MiniClusterUtil; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; -import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.common.testutils.minicluster.MiniClusterUtil; +import org.apache.hudi.hadoop.hive.HoodieCombineHiveInputFormat; import org.apache.hudi.hadoop.testutils.InputFormatTestUtil; import org.apache.avro.Schema; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java index 8fec479f5..3e63fef6c 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java @@ -23,11 +23,11 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.hadoop.testutils.InputFormatTestUtil; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java index 0aa89dd18..0c6ed7ba5 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java @@ -22,12 +22,12 @@ import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; -import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.hadoop.HoodieHiveUtil; import org.apache.avro.Schema; diff --git a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index b4fa186b5..bf383c41e 100644 --- a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -19,11 +19,11 @@ package org.apache.hudi.hive; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.hive.HoodieHiveClient.PartitionEvent; import org.apache.hudi.hive.HoodieHiveClient.PartitionEvent.PartitionEventType; -import org.apache.hudi.hive.testutils.TestUtil; +import org.apache.hudi.hive.testutils.HiveTestUtil; import org.apache.hudi.hive.util.HiveSchemaUtil; import org.apache.hadoop.hive.metastore.api.Partition; @@ -60,17 +60,17 @@ public class TestHiveSyncTool { @BeforeEach public void setUp() throws IOException, InterruptedException { - TestUtil.setUp(); + HiveTestUtil.setUp(); } @AfterEach public void teardown() throws IOException { - TestUtil.clear(); + HiveTestUtil.clear(); } @AfterAll public static void cleanUpClass() { - TestUtil.shutdown(); + HiveTestUtil.shutdown(); } /** @@ -154,51 +154,51 @@ public class TestHiveSyncTool { @ParameterizedTest @MethodSource({"useJdbcAndSchemaFromCommitMetadata"}) public void testBasicSync(boolean useJdbc, boolean useSchemaFromCommitMetadata) throws Exception { - TestUtil.hiveSyncConfig.useJdbc = useJdbc; + HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; - TestUtil.createCOWTable(instantTime, 5, useSchemaFromCommitMetadata); + HiveTestUtil.createCOWTable(instantTime, 5, useSchemaFromCommitMetadata); HoodieHiveClient hiveClient = - new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertFalse(hiveClient.doesTableExist(TestUtil.hiveSyncConfig.tableName), - "Table " + TestUtil.hiveSyncConfig.tableName + " should not exist initially"); + new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); + assertFalse(hiveClient.doesTableExist(HiveTestUtil.hiveSyncConfig.tableName), + "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should not exist initially"); // Lets do the sync - HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - assertTrue(hiveClient.doesTableExist(TestUtil.hiveSyncConfig.tableName), - "Table " + TestUtil.hiveSyncConfig.tableName + " should exist after sync completes"); - assertEquals(hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).size(), + assertTrue(hiveClient.doesTableExist(HiveTestUtil.hiveSyncConfig.tableName), + "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should exist after sync completes"); + assertEquals(hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).size(), hiveClient.getDataSchema().getColumns().size() + 1, "Hive Schema should match the table schema + partition field"); - assertEquals(5, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + assertEquals(5, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "Table partitions should match the number of partitions we wrote"); - assertEquals(instantTime, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + assertEquals(instantTime, hiveClient.getLastCommitTimeSynced(HiveTestUtil.hiveSyncConfig.tableName).get(), "The last commit that was sycned should be updated in the TBLPROPERTIES"); // Adding of new partitions List newPartition = Arrays.asList("2050/01/01"); - hiveClient.addPartitionsToTable(TestUtil.hiveSyncConfig.tableName, Arrays.asList()); - assertEquals(5, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + hiveClient.addPartitionsToTable(HiveTestUtil.hiveSyncConfig.tableName, Arrays.asList()); + assertEquals(5, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "No new partition should be added"); - hiveClient.addPartitionsToTable(TestUtil.hiveSyncConfig.tableName, newPartition); - assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + hiveClient.addPartitionsToTable(HiveTestUtil.hiveSyncConfig.tableName, newPartition); + assertEquals(6, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "New partition should be added"); // Update partitions - hiveClient.updatePartitionsToTable(TestUtil.hiveSyncConfig.tableName, Arrays.asList()); - assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + hiveClient.updatePartitionsToTable(HiveTestUtil.hiveSyncConfig.tableName, Arrays.asList()); + assertEquals(6, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "Partition count should remain the same"); - hiveClient.updatePartitionsToTable(TestUtil.hiveSyncConfig.tableName, newPartition); - assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + hiveClient.updatePartitionsToTable(HiveTestUtil.hiveSyncConfig.tableName, newPartition); + assertEquals(6, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "Partition count should remain the same"); // Alter partitions // Manually change a hive partition location to check if the sync will detect // it and generage a partition update event for it. - hiveClient.updateHiveSQL("ALTER TABLE `" + TestUtil.hiveSyncConfig.tableName + hiveClient.updateHiveSQL("ALTER TABLE `" + HiveTestUtil.hiveSyncConfig.tableName + "` PARTITION (`datestr`='2050-01-01') SET LOCATION '/some/new/location'"); - hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - List hivePartitions = hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName); + hiveClient = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); + List hivePartitions = hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName); List writtenPartitionsSince = hiveClient.getPartitionsWrittenToSince(Option.empty()); writtenPartitionsSince.add(newPartition.get(0)); List partitionEvents = hiveClient.getPartitionEvents(hivePartitions, writtenPartitionsSince); @@ -206,119 +206,119 @@ public class TestHiveSyncTool { assertEquals(PartitionEventType.UPDATE, partitionEvents.iterator().next().eventType, "The one partition event must of type UPDATE"); - tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); // Sync should update the changed partition to correct path - List tablePartitions = hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName); + List tablePartitions = hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName); assertEquals(6, tablePartitions.size(), "The one partition we wrote should be added to hive"); - assertEquals(instantTime, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + assertEquals(instantTime, hiveClient.getLastCommitTimeSynced(HiveTestUtil.hiveSyncConfig.tableName).get(), "The last commit that was sycned should be 100"); } @ParameterizedTest @MethodSource("useJdbc") public void testSyncIncremental(boolean useJdbc) throws Exception { - TestUtil.hiveSyncConfig.useJdbc = useJdbc; + HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; String commitTime1 = "100"; - TestUtil.createCOWTable(commitTime1, 5, true); + HiveTestUtil.createCOWTable(commitTime1, 5, true); HoodieHiveClient hiveClient = - new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); // Lets do the sync - HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - assertEquals(5, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + assertEquals(5, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "Table partitions should match the number of partitions we wrote"); - assertEquals(commitTime1, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + assertEquals(commitTime1, hiveClient.getLastCommitTimeSynced(HiveTestUtil.hiveSyncConfig.tableName).get(), "The last commit that was sycned should be updated in the TBLPROPERTIES"); // Now lets create more parititions and these are the only ones which needs to be synced DateTime dateTime = DateTime.now().plusDays(6); String commitTime2 = "101"; - TestUtil.addCOWPartitions(1, true, true, dateTime, commitTime2); + HiveTestUtil.addCOWPartitions(1, true, true, dateTime, commitTime2); // Lets do the sync - hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + hiveClient = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); List writtenPartitionsSince = hiveClient.getPartitionsWrittenToSince(Option.of(commitTime1)); assertEquals(1, writtenPartitionsSince.size(), "We should have one partition written after 100 commit"); - List hivePartitions = hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName); + List hivePartitions = hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName); List partitionEvents = hiveClient.getPartitionEvents(hivePartitions, writtenPartitionsSince); assertEquals(1, partitionEvents.size(), "There should be only one paritition event"); assertEquals(PartitionEventType.ADD, partitionEvents.iterator().next().eventType, "The one partition event must of type ADD"); - tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); // Sync should add the one partition - assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + assertEquals(6, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "The one partition we wrote should be added to hive"); - assertEquals(commitTime2, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + assertEquals(commitTime2, hiveClient.getLastCommitTimeSynced(HiveTestUtil.hiveSyncConfig.tableName).get(), "The last commit that was sycned should be 101"); } @ParameterizedTest @MethodSource("useJdbc") public void testSyncIncrementalWithSchemaEvolution(boolean useJdbc) throws Exception { - TestUtil.hiveSyncConfig.useJdbc = useJdbc; + HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; String commitTime1 = "100"; - TestUtil.createCOWTable(commitTime1, 5, true); + HiveTestUtil.createCOWTable(commitTime1, 5, true); HoodieHiveClient hiveClient = - new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); // Lets do the sync - HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - int fields = hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).size(); + int fields = hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).size(); // Now lets create more parititions and these are the only ones which needs to be synced DateTime dateTime = DateTime.now().plusDays(6); String commitTime2 = "101"; - TestUtil.addCOWPartitions(1, false, true, dateTime, commitTime2); + HiveTestUtil.addCOWPartitions(1, false, true, dateTime, commitTime2); // Lets do the sync - tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - assertEquals(fields + 3, hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).size(), + assertEquals(fields + 3, hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).size(), "Hive Schema has evolved and should not be 3 more field"); - assertEquals("BIGINT", hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).get("favorite_number"), + assertEquals("BIGINT", hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).get("favorite_number"), "Hive Schema has evolved - Field favorite_number has evolved from int to long"); - assertTrue(hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).containsKey("favorite_movie"), + assertTrue(hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).containsKey("favorite_movie"), "Hive Schema has evolved - Field favorite_movie was added"); // Sync should add the one partition - assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + assertEquals(6, hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), "The one partition we wrote should be added to hive"); - assertEquals(commitTime2, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + assertEquals(commitTime2, hiveClient.getLastCommitTimeSynced(HiveTestUtil.hiveSyncConfig.tableName).get(), "The last commit that was sycned should be 101"); } @ParameterizedTest @MethodSource("useJdbcAndSchemaFromCommitMetadata") public void testSyncMergeOnRead(boolean useJdbc, boolean useSchemaFromCommitMetadata) throws Exception { - TestUtil.hiveSyncConfig.useJdbc = useJdbc; + HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; String deltaCommitTime = "101"; - TestUtil.createMORTable(instantTime, deltaCommitTime, 5, true, - useSchemaFromCommitMetadata); + HiveTestUtil.createMORTable(instantTime, deltaCommitTime, 5, true, + useSchemaFromCommitMetadata); - String roTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_READ_OPTIMIZED_TABLE; - HoodieHiveClient hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertFalse(hiveClient.doesTableExist(roTableName), "Table " + TestUtil.hiveSyncConfig.tableName + " should not exist initially"); + String roTableName = HiveTestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_READ_OPTIMIZED_TABLE; + HoodieHiveClient hiveClient = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); + assertFalse(hiveClient.doesTableExist(roTableName), "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should not exist initially"); // Lets do the sync - HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); assertTrue(hiveClient.doesTableExist(roTableName), "Table " + roTableName + " should exist after sync completes"); if (useSchemaFromCommitMetadata) { assertEquals(hiveClient.getTableSchema(roTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size() - + HoodieRecord.HOODIE_META_COLUMNS.size(), + SchemaTestUtil.getSimpleSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size() + + HoodieRecord.HOODIE_META_COLUMNS.size(), "Hive Schema should match the table schema + partition field"); } else { // The data generated and schema in the data file do not have metadata columns, so we need a separate check. assertEquals(hiveClient.getTableSchema(roTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size(), - "Hive Schema should match the table schema + partition field"); + SchemaTestUtil.getSimpleSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size(), + "Hive Schema should match the table schema + partition field"); } assertEquals(5, hiveClient.scanTablePartitions(roTableName).size(), @@ -331,24 +331,24 @@ public class TestHiveSyncTool { String commitTime2 = "102"; String deltaCommitTime2 = "103"; - TestUtil.addCOWPartitions(1, true, useSchemaFromCommitMetadata, dateTime, commitTime2); - TestUtil.addMORPartitions(1, true, false, + HiveTestUtil.addCOWPartitions(1, true, useSchemaFromCommitMetadata, dateTime, commitTime2); + HiveTestUtil.addMORPartitions(1, true, false, useSchemaFromCommitMetadata, dateTime, commitTime2, deltaCommitTime2); // Lets do the sync - tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + hiveClient = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); if (useSchemaFromCommitMetadata) { assertEquals(hiveClient.getTableSchema(roTableName).size(), - SchemaTestUtil.getEvolvedSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size() - + HoodieRecord.HOODIE_META_COLUMNS.size(), + SchemaTestUtil.getEvolvedSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size() + + HoodieRecord.HOODIE_META_COLUMNS.size(), "Hive Schema should match the evolved table schema + partition field"); } else { // The data generated and schema in the data file do not have metadata columns, so we need a separate check. assertEquals(hiveClient.getTableSchema(roTableName).size(), - SchemaTestUtil.getEvolvedSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size(), - "Hive Schema should match the evolved table schema + partition field"); + SchemaTestUtil.getEvolvedSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size(), + "Hive Schema should match the evolved table schema + partition field"); } // Sync should add the one partition assertEquals(6, hiveClient.scanTablePartitions(roTableName).size(), @@ -360,36 +360,36 @@ public class TestHiveSyncTool { @ParameterizedTest @MethodSource("useJdbcAndSchemaFromCommitMetadata") public void testSyncMergeOnReadRT(boolean useJdbc, boolean useSchemaFromCommitMetadata) throws Exception { - TestUtil.hiveSyncConfig.useJdbc = useJdbc; + HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; String deltaCommitTime = "101"; - String snapshotTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE; - TestUtil.createMORTable(instantTime, deltaCommitTime, 5, true, useSchemaFromCommitMetadata); + String snapshotTableName = HiveTestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE; + HiveTestUtil.createMORTable(instantTime, deltaCommitTime, 5, true, useSchemaFromCommitMetadata); HoodieHiveClient hiveClientRT = - new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); assertFalse(hiveClientRT.doesTableExist(snapshotTableName), - "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + "Table " + HiveTestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + " should not exist initially"); // Lets do the sync - HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); assertTrue(hiveClientRT.doesTableExist(snapshotTableName), - "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + "Table " + HiveTestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + " should exist after sync completes"); if (useSchemaFromCommitMetadata) { assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size() - + HoodieRecord.HOODIE_META_COLUMNS.size(), + SchemaTestUtil.getSimpleSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size() + + HoodieRecord.HOODIE_META_COLUMNS.size(), "Hive Schema should match the table schema + partition field"); } else { // The data generated and schema in the data file do not have metadata columns, so we need a separate check. assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size(), - "Hive Schema should match the table schema + partition field"); + SchemaTestUtil.getSimpleSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size(), + "Hive Schema should match the table schema + partition field"); } assertEquals(5, hiveClientRT.scanTablePartitions(snapshotTableName).size(), @@ -402,23 +402,23 @@ public class TestHiveSyncTool { String commitTime2 = "102"; String deltaCommitTime2 = "103"; - TestUtil.addCOWPartitions(1, true, useSchemaFromCommitMetadata, dateTime, commitTime2); - TestUtil.addMORPartitions(1, true, false, useSchemaFromCommitMetadata, dateTime, commitTime2, deltaCommitTime2); + HiveTestUtil.addCOWPartitions(1, true, useSchemaFromCommitMetadata, dateTime, commitTime2); + HiveTestUtil.addMORPartitions(1, true, false, useSchemaFromCommitMetadata, dateTime, commitTime2, deltaCommitTime2); // Lets do the sync - tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - hiveClientRT = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + hiveClientRT = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); if (useSchemaFromCommitMetadata) { assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getEvolvedSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size() - + HoodieRecord.HOODIE_META_COLUMNS.size(), - "Hive Schema should match the evolved table schema + partition field"); + SchemaTestUtil.getEvolvedSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size() + + HoodieRecord.HOODIE_META_COLUMNS.size(), + "Hive Schema should match the evolved table schema + partition field"); } else { // The data generated and schema in the data file do not have metadata columns, so we need a separate check. assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getEvolvedSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size(), - "Hive Schema should match the evolved table schema + partition field"); + SchemaTestUtil.getEvolvedSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size(), + "Hive Schema should match the evolved table schema + partition field"); } // Sync should add the one partition assertEquals(6, hiveClientRT.scanTablePartitions(snapshotTableName).size(), @@ -430,21 +430,21 @@ public class TestHiveSyncTool { @ParameterizedTest @MethodSource("useJdbc") public void testMultiPartitionKeySync(boolean useJdbc) throws Exception { - TestUtil.hiveSyncConfig.useJdbc = useJdbc; + HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; - TestUtil.createCOWTable(instantTime, 5, true); + HiveTestUtil.createCOWTable(instantTime, 5, true); - HiveSyncConfig hiveSyncConfig = HiveSyncConfig.copy(TestUtil.hiveSyncConfig); + HiveSyncConfig hiveSyncConfig = HiveSyncConfig.copy(HiveTestUtil.hiveSyncConfig); hiveSyncConfig.partitionValueExtractorClass = MultiPartKeysValueExtractor.class.getCanonicalName(); hiveSyncConfig.tableName = "multi_part_key"; hiveSyncConfig.partitionFields = Arrays.asList("year", "month", "day"); - TestUtil.getCreatedTablesSet().add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName); + HiveTestUtil.getCreatedTablesSet().add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName); - HoodieHiveClient hiveClient = new HoodieHiveClient(hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HoodieHiveClient hiveClient = new HoodieHiveClient(hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); assertFalse(hiveClient.doesTableExist(hiveSyncConfig.tableName), "Table " + hiveSyncConfig.tableName + " should not exist initially"); // Lets do the sync - HiveSyncTool tool = new HiveSyncTool(hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HiveSyncTool tool = new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); assertTrue(hiveClient.doesTableExist(hiveSyncConfig.tableName), "Table " + hiveSyncConfig.tableName + " should exist after sync completes"); @@ -460,27 +460,27 @@ public class TestHiveSyncTool { @ParameterizedTest @MethodSource("useJdbc") public void testReadSchemaForMOR(boolean useJdbc) throws Exception { - TestUtil.hiveSyncConfig.useJdbc = useJdbc; + HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; String commitTime = "100"; - String snapshotTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE; - TestUtil.createMORTable(commitTime, "", 5, false, true); + String snapshotTableName = HiveTestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE; + HiveTestUtil.createMORTable(commitTime, "", 5, false, true); HoodieHiveClient hiveClientRT = - new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - assertFalse(hiveClientRT.doesTableExist(snapshotTableName), "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + assertFalse(hiveClientRT.doesTableExist(snapshotTableName), "Table " + HiveTestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + " should not exist initially"); // Lets do the sync - HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - assertTrue(hiveClientRT.doesTableExist(snapshotTableName), "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + assertTrue(hiveClientRT.doesTableExist(snapshotTableName), "Table " + HiveTestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + " should exist after sync completes"); // Schema being read from compacted base files assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size() - + HoodieRecord.HOODIE_META_COLUMNS.size(), + SchemaTestUtil.getSimpleSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size() + + HoodieRecord.HOODIE_META_COLUMNS.size(), "Hive Schema should match the table schema + partition field"); assertEquals(5, hiveClientRT.scanTablePartitions(snapshotTableName).size(), "Table partitions should match the number of partitions we wrote"); @@ -489,16 +489,16 @@ public class TestHiveSyncTool { String commitTime2 = "102"; String deltaCommitTime2 = "103"; - TestUtil.addMORPartitions(1, true, false, true, dateTime, commitTime2, deltaCommitTime2); + HiveTestUtil.addMORPartitions(1, true, false, true, dateTime, commitTime2, deltaCommitTime2); // Lets do the sync - tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); tool.syncHoodieTable(); - hiveClientRT = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); + hiveClientRT = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); // Schema being read from the log files assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getEvolvedSchema().getFields().size() + TestUtil.hiveSyncConfig.partitionFields.size() - + HoodieRecord.HOODIE_META_COLUMNS.size(), + SchemaTestUtil.getEvolvedSchema().getFields().size() + HiveTestUtil.hiveSyncConfig.partitionFields.size() + + HoodieRecord.HOODIE_META_COLUMNS.size(), "Hive Schema should match the evolved table schema + partition field"); // Sync should add the one partition assertEquals(6, hiveClientRT.scanTablePartitions(snapshotTableName).size(), "The 1 partition we wrote should be added to hive"); diff --git a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java index e3a70a862..725b6134e 100644 --- a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java +++ b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java @@ -18,7 +18,7 @@ package org.apache.hudi.hive.testutils; -import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/TestUtil.java b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java similarity index 98% rename from hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/TestUtil.java rename to hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index 47d10956a..d0d1b667a 100644 --- a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/TestUtil.java +++ b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -23,8 +23,6 @@ import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.bloom.BloomFilterTypeCode; import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.minicluster.HdfsTestService; -import org.apache.hudi.common.minicluster.ZookeeperTestService; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCommitMetadata; @@ -39,8 +37,10 @@ import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.common.testutils.minicluster.HdfsTestService; +import org.apache.hudi.common.testutils.minicluster.ZookeeperTestService; import org.apache.hudi.common.util.FileIOUtils; -import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HiveSyncTool; import org.apache.hudi.hive.HoodieHiveClient; @@ -80,7 +80,7 @@ import java.util.UUID; import static org.junit.jupiter.api.Assertions.fail; @SuppressWarnings("SameParameterValue") -public class TestUtil { +public class HiveTestUtil { private static MiniDFSCluster dfsCluster; private static ZooKeeperServer zkServer; diff --git a/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala b/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala index 81190ceb3..a1229fbf5 100644 --- a/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala +++ b/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala @@ -20,7 +20,8 @@ package org.apache.hudi import org.apache.avro.generic.GenericRecord import org.apache.hudi.common.config.TypedProperties import org.apache.hudi.common.model.{EmptyHoodieRecordPayload, OverwriteWithLatestAvroPayload} -import org.apache.hudi.common.util.{Option, SchemaTestUtil} +import org.apache.hudi.common.testutils.SchemaTestUtil +import org.apache.hudi.common.util.Option import org.apache.hudi.exception.{HoodieException, HoodieKeyException} import org.apache.hudi.keygen.{ComplexKeyGenerator, GlobalDeleteKeyGenerator, SimpleKeyGenerator} import org.junit.jupiter.api.Assertions.assertEquals diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java index 9aca63575..2b2d68870 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java @@ -19,8 +19,8 @@ package org.apache.hudi.utilities.checkpointing; import org.apache.hudi.common.config.TypedProperties; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.exception.HoodieException; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java index a8474f670..e1299ce2b 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java @@ -22,10 +22,10 @@ import org.apache.hudi.client.HoodieReadClient; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.minicluster.HdfsTestService; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.minicluster.HdfsTestService; import org.apache.hudi.utilities.HDFSParquetImporter; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java index b0347f51c..e0b5ac3d1 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java @@ -20,8 +20,8 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.utilities.HoodieSnapshotCopier; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java index 4e6384398..327b0ec71 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java @@ -19,10 +19,10 @@ package org.apache.hudi.utilities.inline.fs; import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.fs.inline.FileSystemTestUtils; import org.apache.hudi.common.fs.inline.InLineFileSystem; import org.apache.hudi.common.fs.inline.InMemoryFileSystem; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.testutils.FileSystemTestUtils; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; @@ -42,9 +42,9 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.FILE_SCHEME; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.getPhantomFile; -import static org.apache.hudi.common.fs.inline.FileSystemTestUtils.getRandomOuterInMemPath; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.FILE_SCHEME; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.getPhantomFile; +import static org.apache.hudi.common.testutils.FileSystemTestUtils.getRandomOuterInMemPath; import static org.junit.jupiter.api.Assertions.assertArrayEquals; /** diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/keygen/TestTimestampBasedKeyGenerator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/keygen/TestTimestampBasedKeyGenerator.java index 05cb6f9fa..81f77510e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/keygen/TestTimestampBasedKeyGenerator.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/keygen/TestTimestampBasedKeyGenerator.java @@ -21,7 +21,7 @@ package org.apache.hudi.utilities.keygen; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieKey; -import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java index ab9b5dfee..81bff4271 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java @@ -21,11 +21,11 @@ package org.apache.hudi.utilities.testutils; import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.config.TypedProperties; -import org.apache.hudi.common.minicluster.HdfsTestService; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.minicluster.HdfsTestService; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException;