From 742c2040995167871db976fc0eb280347401ffc4 Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Mon, 1 Jun 2020 19:25:42 -0700 Subject: [PATCH] [HUDI-811] Restructure test packages in hudi-client/cli (#1689) --- .../commands/TestArchivedCommitsCommand.java | 6 +- .../hudi/cli/commands/TestCleansCommand.java | 4 +- .../commands/TestFileSystemViewCommand.java | 4 +- .../commands/TestHoodieLogFileCommand.java | 4 +- .../hudi/cli/commands/TestRepairsCommand.java | 14 +- .../cli/commands/TestRollbacksCommand.java | 4 +- .../cli/commands/TestSavepointsCommand.java | 5 +- .../cli/commands/TestSparkEnvCommand.java | 2 +- .../hudi/cli/commands/TestStatsCommand.java | 12 +- .../hudi/cli/commands/TestTableCommand.java | 2 +- .../hudi/cli/integ/ITTestCleansCommand.java | 4 +- .../integ/ITTestHDFSParquetImportCommand.java | 15 +- .../hudi/cli/integ/ITTestRepairsCommand.java | 6 +- .../cli/integ/ITTestSavepointsCommand.java | 5 +- .../AbstractShellIntegrationTest.java | 4 +- .../HoodieTestCommitMetadataGenerator.java | 4 +- .../HoodieTestCommitUtilities.java | 2 +- .../apache/hudi/client/HoodieWriteClient.java | 7 +- .../hudi/client/TestClientRollback.java | 5 +- .../client/TestCompactionAdminClient.java | 3 +- .../TestHoodieClientOnCopyOnWriteStorage.java | 13 +- .../hudi/client/TestHoodieReadClient.java | 3 +- .../org/apache/hudi/client/TestMultiFS.java | 6 +- .../hudi/client/TestTableSchemaEvolution.java | 23 +-- .../client/TestUpdateSchemaEvolution.java | 4 +- .../config/TestHoodieWriteConfig.java | 4 +- .../execution/TestBoundedInMemoryQueue.java | 4 +- ... => TestSparkBoundedInMemoryExecutor.java} | 6 +- .../apache/hudi/index/TestHoodieIndex.java | 8 +- .../index/bloom/TestHoodieBloomIndex.java | 6 +- .../bloom/TestHoodieGlobalBloomIndex.java | 6 +- .../TestHBaseIndex.java} | 17 ++- .../TestHBaseQPSResourceAllocator.java | 10 +- .../hudi/io/TestHoodieCommitArchiveLog.java | 4 +- .../io/TestHoodieKeyLocationFetchHandle.java | 10 +- .../apache/hudi/io/TestHoodieMergeHandle.java | 6 +- .../TestHoodieStorageWriterFactory.java | 6 +- .../org/apache/hudi/table/TestCleaner.java | 8 +- .../hudi/table/TestConsistencyGuard.java | 4 +- ...e.java => TestHoodieMergeOnReadTable.java} | 14 +- .../commit/TestCopyOnWriteActionExecutor.java | 8 +- .../action/commit/TestUpsertPartitioner.java | 6 +- .../action/compact/TestAsyncCompaction.java | 10 +- .../action/compact/TestHoodieCompactor.java | 4 +- .../HoodieClientTestBase.java} | 131 +++++++++--------- .../HoodieClientTestHarness.java | 11 +- .../HoodieClientTestUtils.java | 2 +- .../HoodieMergeOnReadTestUtils.java | 2 +- .../HoodieTestDataGenerator.java | 4 +- .../TestRawTripPayload.java | 2 +- hudi-spark/src/test/java/HoodieJavaApp.java | 4 +- .../src/test/java/HoodieJavaStreamingApp.java | 2 +- .../hudi/testutils/DataSourceTestUtils.java | 1 - .../hudi/functional/TestDataSource.scala | 3 +- .../functional/TestHDFSParquetImporter.java | 4 +- .../functional/TestHoodieDeltaStreamer.java | 2 +- .../TestHoodieMultiTableDeltaStreamer.java | 2 +- .../functional/TestHoodieSnapshotCopier.java | 2 +- .../TestHoodieSnapshotExporter.java | 4 +- .../inline/fs/TestParquetInLining.java | 2 +- .../utilities/sources/TestKafkaSource.java | 2 +- .../testutils/UtilitiesTestBase.java | 4 +- .../sources/AbstractBaseTestSource.java | 2 +- .../sources/AbstractDFSSourceTestBase.java | 2 +- 64 files changed, 241 insertions(+), 249 deletions(-) rename hudi-cli/src/test/java/org/apache/hudi/cli/{ => testutils}/AbstractShellIntegrationTest.java (94%) rename hudi-cli/src/test/java/org/apache/hudi/cli/{common => testutils}/HoodieTestCommitMetadataGenerator.java (98%) rename hudi-cli/src/test/java/org/apache/hudi/cli/{common => testutils}/HoodieTestCommitUtilities.java (98%) rename hudi-client/src/test/java/org/apache/hudi/{common => }/config/TestHoodieWriteConfig.java (94%) rename hudi-client/src/test/java/org/apache/hudi/execution/{TestBoundedInMemoryExecutor.java => TestSparkBoundedInMemoryExecutor.java} (94%) rename hudi-client/src/test/java/org/apache/hudi/index/{TestHbaseIndex.java => hbase/TestHBaseIndex.java} (98%) rename hudi-client/src/test/java/org/apache/hudi/index/{ => hbase}/TestHBaseQPSResourceAllocator.java (94%) rename hudi-client/src/test/java/org/apache/hudi/table/{TestMergeOnReadTable.java => TestHoodieMergeOnReadTable.java} (99%) rename hudi-client/src/test/java/org/apache/hudi/{client/TestHoodieClientBase.java => testutils/HoodieClientTestBase.java} (79%) rename hudi-client/src/test/java/org/apache/hudi/{common => testutils}/HoodieClientTestHarness.java (96%) rename hudi-client/src/test/java/org/apache/hudi/{common => testutils}/HoodieClientTestUtils.java (99%) rename hudi-client/src/test/java/org/apache/hudi/{common => testutils}/HoodieMergeOnReadTestUtils.java (99%) rename hudi-client/src/test/java/org/apache/hudi/{common => testutils}/HoodieTestDataGenerator.java (99%) rename hudi-client/src/test/java/org/apache/hudi/{common => testutils}/TestRawTripPayload.java (99%) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java index 88fcb39e4..d25943dcb 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java @@ -18,12 +18,12 @@ package org.apache.hudi.cli.commands; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.TableHeader; -import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; -import org.apache.hudi.cli.common.HoodieTestCommitUtilities; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; +import org.apache.hudi.cli.testutils.HoodieTestCommitUtilities; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java index 6722c9891..849f05300 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java @@ -19,12 +19,12 @@ package org.apache.hudi.cli.commands; import org.apache.hudi.avro.model.HoodieCleanMetadata; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.TableHeader; -import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodieTableType; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java index 83ad7fc23..aec393704 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java @@ -18,12 +18,12 @@ package org.apache.hudi.cli.commands; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.TableHeader; -import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieFileGroup; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java index e490a77e9..e484bef16 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java @@ -19,12 +19,12 @@ package org.apache.hudi.cli.commands; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.TableHeader; -import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java index 9fc49181d..6b72e9a8d 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java @@ -18,21 +18,21 @@ package org.apache.hudi.cli.commands; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; -import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; -import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.junit.jupiter.api.Test; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; import java.io.File; @@ -47,8 +47,8 @@ import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java index d8903aee5..23fbcf26e 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java @@ -19,13 +19,12 @@ package org.apache.hudi.cli.commands; import org.apache.hudi.avro.model.HoodieRollbackMetadata; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.client.HoodieWriteClient; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -36,6 +35,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java index 2c6a3f293..1ec0da4e3 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java @@ -18,14 +18,15 @@ package org.apache.hudi.cli.commands; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; -import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.testutils.HoodieTestDataGenerator; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java index a0c7a497b..ee6f57045 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java @@ -18,8 +18,8 @@ package org.apache.hudi.cli.commands; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java index 3305aa0b4..015697775 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java @@ -18,21 +18,21 @@ package org.apache.hudi.cli.commands; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Snapshot; -import com.codahale.metrics.UniformReservoir; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.TableHeader; -import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; -import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.testutils.HoodieTestDataGenerator; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.UniformReservoir; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java index 7c424df45..006cd45b7 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java @@ -18,8 +18,8 @@ package org.apache.hudi.cli.commands; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCleansCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCleansCommand.java index f76c79b53..1f6f6c7a3 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCleansCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCleansCommand.java @@ -18,10 +18,10 @@ package org.apache.hudi.cli.integ; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.commands.TableCommand; -import org.apache.hudi.cli.common.HoodieTestCommitMetadataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.HoodieInstant; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java index 347396b23..8bc51fc93 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java @@ -18,26 +18,25 @@ package org.apache.hudi.cli.integ; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.commands.TableCommand; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HDFSParquetImporter; import org.apache.hudi.utilities.functional.TestHDFSParquetImporter; import org.apache.hudi.utilities.functional.TestHDFSParquetImporter.HoodieTripModel; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; - -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; import java.io.File; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java index 6af25b6d1..2c1bb9a61 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java @@ -19,12 +19,10 @@ package org.apache.hudi.cli.integ; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.commands.RepairsCommand; import org.apache.hudi.cli.commands.TableCommand; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieLogFile; @@ -34,6 +32,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.testutils.SchemaTestUtil; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileStatus; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java index ee9a18e4b..166f8917a 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java @@ -18,19 +18,18 @@ package org.apache.hudi.cli.integ; -import org.apache.hudi.cli.AbstractShellIntegrationTest; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.commands.TableCommand; -import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; - import org.springframework.shell.core.CommandResult; import java.io.File; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/AbstractShellIntegrationTest.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java similarity index 94% rename from hudi-cli/src/test/java/org/apache/hudi/cli/AbstractShellIntegrationTest.java rename to hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java index 1d96c4b38..ef053f0c1 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/AbstractShellIntegrationTest.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.hudi.cli; +package org.apache.hudi.cli.testutils; -import org.apache.hudi.common.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestHarness; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java similarity index 98% rename from hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java rename to hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java index c239e0d8c..bdf623ec4 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package org.apache.hudi.cli.common; +package org.apache.hudi.cli.testutils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieWriteStat; @@ -28,6 +27,7 @@ import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitUtilities.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitUtilities.java similarity index 98% rename from hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitUtilities.java rename to hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitUtilities.java index 75a02c8b8..8829c8580 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitUtilities.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitUtilities.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.cli.common; +package org.apache.hudi.cli.testutils; import org.apache.hudi.avro.model.HoodieWriteStat; import org.apache.hudi.common.model.HoodieCommitMetadata; diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java index fa0b15c52..950d3f9c5 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java @@ -18,7 +18,6 @@ package org.apache.hudi.client; -import com.codahale.metrics.Timer; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieRestoreMetadata; @@ -52,6 +51,8 @@ import org.apache.hudi.table.UserDefinedBulkInsertPartitioner; import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.compact.CompactHelpers; import org.apache.hudi.table.action.savepoint.SavepointHelpers; + +import com.codahale.metrics.Timer; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; @@ -101,7 +102,7 @@ public class HoodieWriteClient extends AbstractHo this(jsc, clientConfig, rollbackPending, HoodieIndex.createIndex(clientConfig)); } - HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending, HoodieIndex index) { + public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending, HoodieIndex index) { this(jsc, clientConfig, rollbackPending, index, Option.empty()); } @@ -675,4 +676,4 @@ public class HoodieWriteClient extends AbstractHo }); return compactionInstantTimeOpt; } -} \ No newline at end of file +} diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java b/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java index b2976b572..90c824d3c 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java @@ -18,7 +18,6 @@ package org.apache.hudi.client; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCleaningPolicy; @@ -33,6 +32,8 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieRollbackException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.Test; @@ -49,7 +50,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test Cases for rollback of snapshots and commits. */ -public class TestClientRollback extends TestHoodieClientBase { +public class TestClientRollback extends HoodieClientTestBase { /** * Test case for rollback-savepoint interaction. diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java b/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java index a9d45eafb..2d691563e 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java @@ -33,6 +33,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.table.action.compact.OperationResult; +import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -55,7 +56,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -public class TestCompactionAdminClient extends TestHoodieClientBase { +public class TestCompactionAdminClient extends HoodieClientTestBase { private static final Logger LOG = LogManager.getLogger(TestCompactionAdminClient.class); diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java index 66295111d..8c2efed36 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java @@ -18,9 +18,6 @@ package org.apache.hudi.client; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieBaseFile; @@ -49,6 +46,10 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.commit.WriteHelper; +import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.Path; @@ -70,10 +71,10 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import static org.apache.hudi.common.HoodieTestDataGenerator.NULL_SCHEMA; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0; import static org.apache.hudi.common.util.ParquetUtils.readRowKeysFromParquet; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.NULL_SCHEMA; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -82,7 +83,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @SuppressWarnings("unchecked") -public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { +public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { private static final Logger LOG = LogManager.getLogger(TestHoodieClientOnCopyOnWriteStorage.class); diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java index bbadaf41d..a59c1f48d 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; @@ -43,7 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; /** * Test-cases for covering HoodieReadClient APIs */ -public class TestHoodieReadClient extends TestHoodieClientBase { +public class TestHoodieReadClient extends HoodieClientTestBase { /** * Test ReadFilter API after writing new records using HoodieWriteClient.insert. diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java index ad8b87f2c..53c615eab 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java @@ -18,9 +18,6 @@ package org.apache.hudi.client; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieRecord; @@ -32,6 +29,9 @@ import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java index d3081e8bd..0148bcaeb 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java @@ -19,9 +19,6 @@ package org.apache.hudi.client; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; @@ -34,6 +31,10 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieInsertException; import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.index.HoodieIndex.IndexType; +import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; @@ -45,19 +46,20 @@ import java.io.IOException; import java.util.List; import java.util.stream.Collectors; -import static org.apache.hudi.common.HoodieTestDataGenerator.FARE_NESTED_SCHEMA; -import static org.apache.hudi.common.HoodieTestDataGenerator.MAP_TYPE_SCHEMA; -import static org.apache.hudi.common.HoodieTestDataGenerator.TIP_NESTED_SCHEMA; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_SCHEMA_PREFIX; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_SCHEMA_SUFFIX; import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_1; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.FARE_NESTED_SCHEMA; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.MAP_TYPE_SCHEMA; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.TIP_NESTED_SCHEMA; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_SCHEMA_PREFIX; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_SCHEMA_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -public class TestTableSchemaEvolution extends TestHoodieClientBase { +public class TestTableSchemaEvolution extends HoodieClientTestBase { + private final String initCommitTime = "000"; private HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE; private HoodieTestDataGenerator dataGenEvolved = new HoodieTestDataGenerator(); @@ -133,7 +135,6 @@ public class TestTableSchemaEvolution extends TestHoodieClientBase { @Test public void testMORTable() throws Exception { tableType = HoodieTableType.MERGE_ON_READ; - initMetaClient(); // Create the table HoodieTableMetaClient.initTableType(metaClient.getHadoopConf(), metaClient.getBasePath(), diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java index 303be2ac3..7bd9d18ad 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java @@ -18,8 +18,6 @@ package org.apache.hudi.client; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -32,6 +30,8 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.io.HoodieMergeHandle; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-client/src/test/java/org/apache/hudi/common/config/TestHoodieWriteConfig.java b/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java similarity index 94% rename from hudi-client/src/test/java/org/apache/hudi/common/config/TestHoodieWriteConfig.java rename to hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java index a1904a5dd..360fa054b 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/config/TestHoodieWriteConfig.java +++ b/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java @@ -16,10 +16,8 @@ * limitations under the License. */ -package org.apache.hudi.common.config; +package org.apache.hudi.config; -import org.apache.hudi.config.HoodieCompactionConfig; -import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.HoodieWriteConfig.Builder; import org.junit.jupiter.api.Test; diff --git a/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java b/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java index fd5c273d8..d80c86d21 100644 --- a/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java +++ b/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java @@ -18,8 +18,6 @@ package org.apache.hudi.execution; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.util.DefaultSizeEstimator; @@ -32,6 +30,8 @@ import org.apache.hudi.common.util.queue.FunctionBasedQueueProducer; import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.execution.LazyInsertIterable.HoodieInsertValueGenResult; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryExecutor.java b/hudi-client/src/test/java/org/apache/hudi/execution/TestSparkBoundedInMemoryExecutor.java similarity index 94% rename from hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryExecutor.java rename to hudi-client/src/test/java/org/apache/hudi/execution/TestSparkBoundedInMemoryExecutor.java index 29adf8cef..2deea67a8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/execution/TestSparkBoundedInMemoryExecutor.java @@ -18,14 +18,14 @@ package org.apache.hudi.execution; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.execution.LazyInsertIterable.HoodieInsertValueGenResult; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.AfterEach; @@ -42,7 +42,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class TestBoundedInMemoryExecutor extends HoodieClientTestHarness { +public class TestSparkBoundedInMemoryExecutor extends HoodieClientTestHarness { private final String instantTime = HoodieActiveTimeline.createNewInstantTime(); diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index 58a703176..46256b6fc 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -21,10 +21,6 @@ package org.apache.hudi.index; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.EmptyHoodieRecordPayload; @@ -51,6 +47,10 @@ import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex; import org.apache.hudi.index.hbase.HBaseIndex; import org.apache.hudi.index.simple.HoodieSimpleIndex; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.Schema; import org.apache.hadoop.fs.Path; diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java index 5a85f9c06..c5348afc5 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java @@ -19,9 +19,6 @@ package org.apache.hudi.index.bloom; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.bloom.BloomFilterTypeCode; @@ -36,6 +33,9 @@ import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.HoodieKeyLookupHandle; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.Schema; import org.apache.hadoop.fs.Path; diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java index 8953213a6..1dac30cb1 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java @@ -19,9 +19,6 @@ package org.apache.hudi.index.bloom; import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.EmptyHoodieRecordPayload; import org.apache.hudi.common.model.HoodieKey; @@ -32,6 +29,9 @@ import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.Schema; import org.apache.spark.api.java.JavaPairRDD; diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java similarity index 98% rename from hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java rename to hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java index d1122cff5..e3b9529b0 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java @@ -16,12 +16,10 @@ * limitations under the License. */ -package org.apache.hudi.index; +package org.apache.hudi.index.hbase; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieWriteStat; @@ -32,11 +30,11 @@ import org.apache.hudi.config.HoodieHBaseIndexConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; -import org.apache.hudi.index.hbase.HBaseIndex; +import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.hbase.HBaseIndex.HbasePutBatchSizeCalculator; -import org.apache.hudi.index.hbase.HBaseIndexQPSResourceAllocator; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -81,13 +79,14 @@ import static org.mockito.Mockito.when; * {@link MethodOrderer.Alphanumeric} to make sure the tests run in order. Please alter the order of tests running carefully. */ @TestMethodOrder(MethodOrderer.Alphanumeric.class) -public class TestHbaseIndex extends HoodieClientTestHarness { +public class TestHBaseIndex extends HoodieClientTestHarness { private static HBaseTestingUtility utility; private static Configuration hbaseConfig; private static String tableName = "test_table"; - public TestHbaseIndex() {} + public TestHBaseIndex() { + } @AfterAll public static void clean() throws Exception { @@ -111,7 +110,7 @@ public class TestHbaseIndex extends HoodieClientTestHarness { @BeforeEach public void setUp() throws Exception { // Initialize a local spark env - initSparkContexts("TestHbaseIndex"); + initSparkContexts("TestHBaseIndex"); hadoopConf.addResource(utility.getConfiguration()); // Create a temp folder as the base path diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java similarity index 94% rename from hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java rename to hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java index 0984ff4e1..f85a76356 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java @@ -16,19 +16,17 @@ * limitations under the License. */ -package org.apache.hudi.index; +package org.apache.hudi.index.hbase; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieHBaseIndexConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; -import org.apache.hudi.index.hbase.HBaseIndex; -import org.apache.hudi.index.hbase.HBaseIndexQPSResourceAllocator; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java index 09f4616a4..3bee3e877 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java @@ -18,8 +18,6 @@ package org.apache.hudi.io; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -32,6 +30,8 @@ import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTimelineArchiveLog; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java index c4262921f..cbea30ce3 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java @@ -18,10 +18,6 @@ package org.apache.hudi.io; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieBaseFile; @@ -39,6 +35,10 @@ import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndexUtils; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.spark.api.java.JavaSparkContext; import org.junit.jupiter.api.AfterEach; @@ -58,7 +58,7 @@ import java.util.UUID; import scala.Tuple2; import static java.util.stream.Collectors.toList; -import static org.apache.hudi.common.HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS; import static org.junit.jupiter.api.Assertions.assertEquals; /** diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java index 29adf5efb..160f16978 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java @@ -20,9 +20,6 @@ package org.apache.hudi.io; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieWriteStat; @@ -34,6 +31,9 @@ import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java b/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java index 5d1324cd0..a82756aff 100755 --- a/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageWriterFactory.java @@ -19,10 +19,10 @@ package org.apache.hudi.io.storage; import org.apache.hudi.client.SparkTaskContextSupplier; -import org.apache.hudi.client.TestHoodieClientBase; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; @@ -36,7 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests for {@link HoodieStorageWriterFactory}. */ -public class TestHoodieStorageWriterFactory extends TestHoodieClientBase { +public class TestHoodieStorageWriterFactory extends HoodieClientTestBase { @Test public void testGetStorageWriter() throws IOException { diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java index bc5a43b54..82f911da8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -22,10 +22,8 @@ import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.client.HoodieWriteClient; -import org.apache.hudi.client.TestHoodieClientBase; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.HoodieCleanStat; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; @@ -54,6 +52,8 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -88,10 +88,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test Cleaning related logic. */ -public class TestCleaner extends TestHoodieClientBase { +public class TestCleaner extends HoodieClientTestBase { private static final int BIG_BATCH_INSERT_SIZE = 500; - private static final Logger LOG = LogManager.getLogger(TestHoodieClientBase.class); + private static final Logger LOG = LogManager.getLogger(TestCleaner.class); /** * Helper method to do first batch of insert for clean by versions/commits tests. diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java b/hudi-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java index f079e9423..5021f5e74 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java @@ -18,11 +18,11 @@ package org.apache.hudi.table; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; import org.apache.hudi.common.fs.ConsistencyGuard; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FailSafeConsistencyGuard; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java similarity index 99% rename from hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java rename to hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java index 99773b92e..7913bafd9 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java @@ -21,11 +21,6 @@ package org.apache.hudi.table; import org.apache.hudi.client.HoodieReadClient; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieMergeOnReadTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.TestRawTripPayload.MetadataMergeWriteStatus; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCommitMetadata; @@ -57,6 +52,11 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.table.action.deltacommit.DeleteDeltaCommitActionExecutor; import org.apache.hudi.table.action.deltacommit.DeltaCommitActionExecutor; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieMergeOnReadTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.testutils.TestRawTripPayload.MetadataMergeWriteStatus; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileStatus; @@ -83,12 +83,12 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -public class TestMergeOnReadTable extends HoodieClientTestHarness { +public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { private HoodieParquetInputFormat roInputFormat; private JobConf roJobConf; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java index 3fc260100..513f63ac3 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java @@ -20,10 +20,6 @@ package org.apache.hudi.table.action.commit; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.TestRawTripPayload; -import org.apache.hudi.common.TestRawTripPayload.MetadataMergeWriteStatus; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; @@ -41,6 +37,10 @@ import org.apache.hudi.hadoop.HoodieParquetInputFormat; import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.table.HoodieCopyOnWriteTable; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.TestRawTripPayload; +import org.apache.hudi.testutils.TestRawTripPayload.MetadataMergeWriteStatus; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileStatus; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java index 46a91569e..066c54197 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java @@ -18,9 +18,6 @@ package org.apache.hudi.table.action.commit; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -32,6 +29,9 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieCopyOnWriteTable; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.WorkloadProfile; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java index 9b5312c70..0178dba50 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestAsyncCompaction.java @@ -22,10 +22,7 @@ import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.client.HoodieReadClient; import org.apache.hudi.client.HoodieWriteClient; -import org.apache.hudi.client.TestHoodieClientBase; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieFileGroupId; @@ -48,6 +45,9 @@ import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -61,7 +61,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -70,7 +70,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test Cases for Async Compaction and Ingestion interaction. */ -public class TestAsyncCompaction extends TestHoodieClientBase { +public class TestAsyncCompaction extends HoodieClientTestBase { private HoodieWriteConfig getConfig(Boolean autoCommit) { return getConfigBuilder(autoCommit).build(); diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java index 1c4d26e01..7bb1bf912 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java @@ -21,8 +21,6 @@ package org.apache.hudi.table.action.compact; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieRecord; @@ -43,6 +41,8 @@ import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.bloom.HoodieBloomIndex; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java similarity index 79% rename from hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java rename to hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java index c8f9fdd0c..b418d4f70 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java @@ -16,14 +16,12 @@ * limitations under the License. */ -package org.apache.hudi.client; +package org.apache.hudi.testutils; +import org.apache.hudi.client.HoodieReadClient; +import org.apache.hudi.client.HoodieWriteClient; +import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.HoodieCleanStat; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.TestRawTripPayload; -import org.apache.hudi.common.TestRawTripPayload.MetadataMergeWriteStatus; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.model.EmptyHoodieRecordPayload; import org.apache.hudi.common.model.HoodieKey; @@ -44,6 +42,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.testutils.TestRawTripPayload.MetadataMergeWriteStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -70,9 +69,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Base Class providing setup/cleanup and utility methods for testing Hoodie Client facing tests. */ -public class TestHoodieClientBase extends HoodieClientTestHarness { +public class HoodieClientTestBase extends HoodieClientTestHarness { - private static final Logger LOG = LogManager.getLogger(TestHoodieClientBase.class); + private static final Logger LOG = LogManager.getLogger(HoodieClientTestBase.class); @BeforeEach public void setUp() throws Exception { @@ -84,20 +83,20 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { cleanupResources(); } - protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) { + public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) { return getHoodieWriteClient(cfg, false); } - protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit) { + public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit) { return getHoodieWriteClient(cfg, rollbackInflightCommit, HoodieIndex.createIndex(cfg)); } - protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit, - HoodieIndex index) { + public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit, + HoodieIndex index) { return new HoodieWriteClient(jsc, cfg, rollbackInflightCommit, index); } - protected HoodieReadClient getHoodieReadClient(String basePath) { + public HoodieReadClient getHoodieReadClient(String basePath) { return new HoodieReadClient(jsc, basePath, SQLContext.getOrCreate(jsc.sc())); } @@ -106,11 +105,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * * @return Default Hoodie Write Config for tests */ - protected HoodieWriteConfig getConfig() { + public HoodieWriteConfig getConfig() { return getConfigBuilder().build(); } - protected HoodieWriteConfig getConfig(IndexType indexType) { + public HoodieWriteConfig getConfig(IndexType indexType) { return getConfigBuilder(indexType).build(); } @@ -119,7 +118,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * * @return Config Builder */ - protected HoodieWriteConfig.Builder getConfigBuilder() { + public HoodieWriteConfig.Builder getConfigBuilder() { return getConfigBuilder(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA); } @@ -128,11 +127,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * * @return Config Builder */ - HoodieWriteConfig.Builder getConfigBuilder(IndexType indexType) { + public HoodieWriteConfig.Builder getConfigBuilder(IndexType indexType) { return getConfigBuilder(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA, indexType); } - HoodieWriteConfig.Builder getConfigBuilder(String schemaStr) { + public HoodieWriteConfig.Builder getConfigBuilder(String schemaStr) { return getConfigBuilder(schemaStr, IndexType.BLOOM); } @@ -141,7 +140,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * * @return Config Builder */ - HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, IndexType indexType) { + public HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, IndexType indexType) { return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(schemaStr) .withParallelism(2, 2).withBulkInsertParallelism(2).withFinalizeWriteParallelism(2) .withTimelineLayoutVersion(TimelineLayoutVersion.CURR_VERSION) @@ -155,7 +154,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { .withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE).build()); } - protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWriteConfig config) { + public HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWriteConfig config) { HoodieTable table = HoodieTable.create(metaClient, config, hadoopConf); ((SyncableFileSystemView) (table.getSliceView())).reset(); return table; @@ -173,14 +172,14 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { } } - void assertPartitionMetadataForRecords(List inputRecords, FileSystem fs) throws IOException { + public void assertPartitionMetadataForRecords(List inputRecords, FileSystem fs) throws IOException { Set partitionPathSet = inputRecords.stream() .map(HoodieRecord::getPartitionPath) .collect(Collectors.toSet()); assertPartitionMetadata(partitionPathSet.stream().toArray(String[]::new), fs); } - void assertPartitionMetadataForKeys(List inputKeys, FileSystem fs) throws IOException { + public void assertPartitionMetadataForKeys(List inputKeys, FileSystem fs) throws IOException { Set partitionPathSet = inputKeys.stream() .map(HoodieKey::getPartitionPath) .collect(Collectors.toSet()); @@ -191,10 +190,10 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * Ensure presence of partition meta-data at known depth. * * @param partitionPaths Partition paths to check - * @param fs File System + * @param fs File System * @throws IOException in case of error */ - void assertPartitionMetadata(String[] partitionPaths, FileSystem fs) throws IOException { + public void assertPartitionMetadata(String[] partitionPaths, FileSystem fs) throws IOException { for (String partitionPath : partitionPaths) { assertTrue(HoodiePartitionMetadata.hasPartitionMetadata(fs, new Path(basePath, partitionPath))); HoodiePartitionMetadata pmeta = new HoodiePartitionMetadata(fs, new Path(basePath, partitionPath)); @@ -207,9 +206,9 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * Ensure records have location field set. * * @param taggedRecords Tagged Records - * @param instantTime Commit Timestamp + * @param instantTime Commit Timestamp */ - protected void checkTaggedRecords(List taggedRecords, String instantTime) { + public void checkTaggedRecords(List taggedRecords, String instantTime) { for (HoodieRecord rec : taggedRecords) { assertTrue(rec.isCurrentLocationKnown(), "Record " + rec + " found with no location."); assertEquals(rec.getCurrentLocation().getInstantTime(), instantTime, @@ -222,7 +221,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * * @param records List of Hoodie records */ - void assertNodupesWithinPartition(List> records) { + public void assertNodupesWithinPartition(List> records) { Map> partitionToKeys = new HashMap<>(); for (HoodieRecord r : records) { String key = r.getRecordKey(); @@ -283,13 +282,13 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * Generate wrapper for record generation function for testing Prepped APIs. * * @param isPreppedAPI Flag to indicate if this is for testing prepped-version of APIs - * @param writeConfig Hoodie Write Config - * @param wrapped Actual Records Generation function + * @param writeConfig Hoodie Write Config + * @param wrapped Actual Records Generation function * @return Wrapped Function */ - protected Function2, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI, - HoodieWriteConfig writeConfig, - Function2, String, Integer> wrapped) { + public Function2, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI, + HoodieWriteConfig writeConfig, + Function2, String, Integer> wrapped) { if (isPreppedAPI) { return wrapRecordsGenFunctionForPreppedCalls(writeConfig, wrapped); } else { @@ -301,12 +300,12 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * Generate wrapper for delete key generation function for testing Prepped APIs. * * @param isPreppedAPI Flag to indicate if this is for testing prepped-version of APIs - * @param writeConfig Hoodie Write Config - * @param wrapped Actual Records Generation function + * @param writeConfig Hoodie Write Config + * @param wrapped Actual Records Generation function * @return Wrapped Function */ - Function> generateWrapDeleteKeysFn(boolean isPreppedAPI, - HoodieWriteConfig writeConfig, Function> wrapped) { + public Function> generateWrapDeleteKeysFn(boolean isPreppedAPI, + HoodieWriteConfig writeConfig, Function> wrapped) { if (isPreppedAPI) { return wrapDeleteKeysGenFunctionForPreppedCalls(writeConfig, wrapped); } else { @@ -329,10 +328,10 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * @return RDD of write-status * @throws Exception in case of error */ - JavaRDD insertFirstBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, - String initCommitTime, int numRecordsInThisCommit, - Function3, HoodieWriteClient, JavaRDD, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit) throws Exception { + public JavaRDD insertFirstBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, + String initCommitTime, int numRecordsInThisCommit, + Function3, HoodieWriteClient, JavaRDD, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit) throws Exception { final Function2, String, Integer> recordGenFunction = generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateInserts); @@ -357,10 +356,10 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * @return RDD of write-status * @throws Exception in case of error */ - JavaRDD insertBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, - String initCommitTime, int numRecordsInThisCommit, - Function3, HoodieWriteClient, JavaRDD, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { + public JavaRDD insertBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, + String initCommitTime, int numRecordsInThisCommit, + Function3, HoodieWriteClient, JavaRDD, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { final Function2, String, Integer> recordGenFunction = generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateInserts); @@ -387,11 +386,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * @return RDD of write-status * @throws Exception in case of error */ - JavaRDD updateBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, - String prevCommitTime, Option> commitTimesBetweenPrevAndNew, String initCommitTime, - int numRecordsInThisCommit, - Function3, HoodieWriteClient, JavaRDD, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { + public JavaRDD updateBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, + String prevCommitTime, Option> commitTimesBetweenPrevAndNew, String initCommitTime, + int numRecordsInThisCommit, + Function3, HoodieWriteClient, JavaRDD, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { final Function2, String, Integer> recordGenFunction = generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateUniqueUpdates); @@ -417,11 +416,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * @return RDD of write-status * @throws Exception in case of error */ - JavaRDD deleteBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, - String prevCommitTime, String initCommitTime, - int numRecordsInThisCommit, - Function3, HoodieWriteClient, JavaRDD, String> deleteFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception { + public JavaRDD deleteBatch(HoodieWriteConfig writeConfig, HoodieWriteClient client, String newCommitTime, + String prevCommitTime, String initCommitTime, + int numRecordsInThisCommit, + Function3, HoodieWriteClient, JavaRDD, String> deleteFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception { final Function> keyGenFunction = generateWrapDeleteKeysFn(isPreppedAPI, writeConfig, dataGen::generateUniqueDeletes); @@ -447,11 +446,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * @param expTotalCommits Expected number of commits (including this commit) * @throws Exception in case of error */ - JavaRDD writeBatch(HoodieWriteClient client, String newCommitTime, String prevCommitTime, - Option> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, - Function2, String, Integer> recordGenFunction, - Function3, HoodieWriteClient, JavaRDD, String> writeFn, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { + public JavaRDD writeBatch(HoodieWriteClient client, String newCommitTime, String prevCommitTime, + Option> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, + Function2, String, Integer> recordGenFunction, + Function3, HoodieWriteClient, JavaRDD, String> writeFn, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { // Write 1 (only inserts) client.startCommitWithTime(newCommitTime); @@ -515,11 +514,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * @param expTotalRecords Expected number of records when scanned * @throws Exception in case of error */ - JavaRDD deleteBatch(HoodieWriteClient client, String newCommitTime, String prevCommitTime, - String initCommitTime, int numRecordsInThisCommit, - Function> keyGenFunction, - Function3, HoodieWriteClient, JavaRDD, String> deleteFn, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception { + public JavaRDD deleteBatch(HoodieWriteClient client, String newCommitTime, String prevCommitTime, + String initCommitTime, int numRecordsInThisCommit, + Function> keyGenFunction, + Function3, HoodieWriteClient, JavaRDD, String> deleteFn, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception { // Delete 1 (only deletes) client.startCommitWithTime(newCommitTime); @@ -567,10 +566,10 @@ public class TestHoodieClientBase extends HoodieClientTestHarness { * Get Cleaner state corresponding to a partition path. * * @param hoodieCleanStatsTwo List of Clean Stats - * @param partitionPath Partition path for filtering + * @param partitionPath Partition path for filtering * @return Cleaner state corresponding to partition path */ - protected HoodieCleanStat getCleanStat(List hoodieCleanStatsTwo, String partitionPath) { + public HoodieCleanStat getCleanStat(List hoodieCleanStatsTwo, String partitionPath) { return hoodieCleanStatsTwo.stream().filter(e -> e.getPartitionPath().equals(partitionPath)).findFirst().orElse(null); } diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestHarness.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java similarity index 96% rename from hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestHarness.java rename to hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java index 2b336695b..111294d9e 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestHarness.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.hudi.common; +package org.apache.hudi.testutils; import org.apache.hudi.client.SparkTaskContextSupplier; -import org.apache.hudi.client.TestHoodieClientBase; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; @@ -71,9 +70,7 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im protected transient DistributedFileSystem dfs; /** - * Initializes resource group for the subclasses of {@link TestHoodieClientBase}. - * - * @throws IOException + * Initializes resource group for the subclasses of {@link HoodieClientTestBase}. */ public void initResources() throws IOException { initPath(); @@ -84,9 +81,7 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im } /** - * Cleanups resource group for the subclasses of {@link TestHoodieClientBase}. - * - * @throws IOException + * Cleanups resource group for the subclasses of {@link HoodieClientTestBase}. */ public void cleanupResources() throws IOException { cleanupMetaClient(); diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java similarity index 99% rename from hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java rename to hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java index 4882469e6..698f901b5 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common; +package org.apache.hudi.testutils; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroWriteSupport; diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java similarity index 99% rename from hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java rename to hudi-client/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java index e54b53b8e..23bd33309 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common; +package org.apache.hudi.testutils; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.testutils.HoodieTestUtils; diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieTestDataGenerator.java similarity index 99% rename from hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java rename to hudi-client/src/test/java/org/apache/hudi/testutils/HoodieTestDataGenerator.java index 9a55ade08..9f7ed2362 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieTestDataGenerator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common; +package org.apache.hudi.testutils; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.model.HoodieCompactionPlan; @@ -707,4 +707,4 @@ public class HoodieTestDataGenerator { public void close() { existingKeysBySchema.clear(); } -} \ No newline at end of file +} diff --git a/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java b/hudi-client/src/test/java/org/apache/hudi/testutils/TestRawTripPayload.java similarity index 99% rename from hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java rename to hudi-client/src/test/java/org/apache/hudi/testutils/TestRawTripPayload.java index 1fd395f9e..31ef912a8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/TestRawTripPayload.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.common; +package org.apache.hudi.testutils; import org.apache.hudi.avro.MercifulJsonConverter; import org.apache.hudi.client.WriteStatus; diff --git a/hudi-spark/src/test/java/HoodieJavaApp.java b/hudi-spark/src/test/java/HoodieJavaApp.java index 0cf316865..a179c4203 100644 --- a/hudi-spark/src/test/java/HoodieJavaApp.java +++ b/hudi-spark/src/test/java/HoodieJavaApp.java @@ -19,8 +19,6 @@ import org.apache.hudi.DataSourceReadOptions; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.HoodieDataSourceHelpers; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.config.HoodieWriteConfig; @@ -29,6 +27,8 @@ import org.apache.hudi.hive.NonPartitionedExtractor; import org.apache.hudi.keygen.NonpartitionedKeyGenerator; import org.apache.hudi.keygen.SimpleKeyGenerator; import org.apache.hudi.testutils.DataSourceTestUtils; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; diff --git a/hudi-spark/src/test/java/HoodieJavaStreamingApp.java b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java index a539eebfe..70ec3fae2 100644 --- a/hudi-spark/src/test/java/HoodieJavaStreamingApp.java +++ b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java @@ -19,11 +19,11 @@ import org.apache.hudi.DataSourceReadOptions; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.HoodieDataSourceHelpers; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.hive.MultiPartKeysValueExtractor; import org.apache.hudi.testutils.DataSourceTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; diff --git a/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java b/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java index de07adcff..130a6e4f0 100644 --- a/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java +++ b/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java @@ -18,7 +18,6 @@ package org.apache.hudi.testutils; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; diff --git a/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala b/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala index 835248575..d2ca589f0 100644 --- a/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala +++ b/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala @@ -18,10 +18,9 @@ package org.apache.hudi.functional import org.apache.hadoop.fs.{FileSystem, Path} -import org.apache.hudi.common.HoodieTestDataGenerator import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.config.HoodieWriteConfig -import org.apache.hudi.testutils.DataSourceTestUtils +import org.apache.hudi.testutils.{DataSourceTestUtils, HoodieTestDataGenerator} import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.spark.sql._ import org.apache.spark.sql.functions.col diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java index e1299ce2b..b396bac50 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java @@ -20,12 +20,12 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.client.HoodieReadClient; import org.apache.hudi.client.HoodieWriteClient; -import org.apache.hudi.common.HoodieClientTestUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.testutils.minicluster.HdfsTestService; +import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HDFSParquetImporter; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java index 512318fb2..3bd887d61 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java @@ -19,7 +19,6 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.DataSourceWriteOptions; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.config.DFSPropertiesConfiguration; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.fs.FSUtils; @@ -38,6 +37,7 @@ import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HoodieHiveClient; import org.apache.hudi.hive.MultiPartKeysValueExtractor; import org.apache.hudi.keygen.SimpleKeyGenerator; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.Operation; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java index 37db4cda6..130f7a8b9 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java @@ -19,9 +19,9 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.DataSourceWriteOptions; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.HoodieMultiTableDeltaStreamer; import org.apache.hudi.utilities.deltastreamer.TableExecutionContext; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java index e0b5ac3d1..2accc4b13 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java @@ -18,10 +18,10 @@ package org.apache.hudi.utilities.functional; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HoodieSnapshotCopier; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java index 89c31e01c..34116a87e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java @@ -19,8 +19,6 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.client.HoodieWriteClient; -import org.apache.hudi.common.HoodieClientTestHarness; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; @@ -28,6 +26,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex.IndexType; +import org.apache.hudi.testutils.HoodieClientTestHarness; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HoodieSnapshotExporter; import org.apache.hudi.utilities.HoodieSnapshotExporter.Config; import org.apache.hudi.utilities.HoodieSnapshotExporter.OutputFormatValidator; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java index 327b0ec71..ce21dd384 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java @@ -18,11 +18,11 @@ package org.apache.hudi.utilities.inline.fs; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.fs.inline.InLineFileSystem; import org.apache.hudi.common.fs.inline.InMemoryFileSystem; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.testutils.FileSystemTestUtils; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java index 0eb15da35..33c0a7f69 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java @@ -19,9 +19,9 @@ package org.apache.hudi.utilities.sources; import org.apache.hudi.AvroConversionUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.util.Option; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.SourceFormatAdapter; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java index 81bff4271..44aaefdab 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java @@ -18,8 +18,6 @@ package org.apache.hudi.utilities.testutils; -import org.apache.hudi.common.HoodieTestDataGenerator; -import org.apache.hudi.common.TestRawTripPayload; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; @@ -32,6 +30,8 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HoodieHiveClient; import org.apache.hudi.hive.testutils.HiveTestService; +import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.hudi.utilities.UtilHelpers; import org.apache.hudi.utilities.sources.TestDataSource; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java index 5d5d16bb7..67bab4df4 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java @@ -18,12 +18,12 @@ package org.apache.hudi.utilities.testutils.sources; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.RocksDBBasedMap; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.AvroSource; import org.apache.hudi.utilities.testutils.sources.config.TestSourceConfig; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java index 0a3809879..e30a183f5 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java @@ -19,9 +19,9 @@ package org.apache.hudi.utilities.testutils.sources; import org.apache.hudi.AvroConversionUtils; -import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.Option; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.SourceFormatAdapter; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.sources.InputBatch;