From df8f099c999379038bf189491ff71dfb958806dd Mon Sep 17 00:00:00 2001 From: Mathieu Date: Wed, 26 Aug 2020 08:49:01 +0800 Subject: [PATCH] [HUDI-532] Add java doc for the test classes of hudi test suite (#1901) --- .../test/java/org/apache/hudi/integ/ITTestBase.java | 3 +++ .../TestDFSHoodieTestSuiteWriterAdapter.java | 3 +++ .../integ/testsuite/TestFileDeltaInputWriter.java | 3 +++ .../testsuite/configuration/TestWorkflowBuilder.java | 3 +++ .../testsuite/converter/TestUpdateConverter.java | 9 +++++++++ .../integ/testsuite/dag/ComplexDagGenerator.java | 9 +++++++++ .../integ/testsuite/dag/HiveSyncDagGenerator.java | 3 +++ .../integ/testsuite/dag/HiveSyncDagGeneratorMOR.java | 4 ++++ .../hudi/integ/testsuite/dag/TestDagUtils.java | 3 +++ .../generator/TestGenericRecordPayloadEstimator.java | 3 +++ .../generator/TestGenericRecordPayloadGenerator.java | 3 +++ .../integ/testsuite/job/TestHoodieTestSuiteJob.java | 3 +++ .../reader/TestDFSAvroDeltaInputReader.java | 3 +++ .../reader/TestDFSHoodieDatasetInputReader.java | 3 +++ .../apache/hudi/integ/testsuite/utils/TestUtils.java | 12 ++++++++++++ 15 files changed, 67 insertions(+) diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java index d0b32eec3..eda776e78 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java @@ -49,6 +49,9 @@ import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; +/** + * Base test class for IT Test helps to run command and generate data. + */ public abstract class ITTestBase { public static final Logger LOG = LogManager.getLogger(ITTestBase.class); diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java index f6d907388..ff41b44dd 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java @@ -52,6 +52,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +/** + * Unit test against DeltaWriterAdapter, by testing writing DFS files. + */ public class TestDFSHoodieTestSuiteWriterAdapter extends UtilitiesTestBase { private FilebasedSchemaProvider schemaProvider; diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java index 741966706..8e175c5bf 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java @@ -48,6 +48,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link DeltaInputWriter}. + */ public class TestFileDeltaInputWriter extends UtilitiesTestBase { private FilebasedSchemaProvider schemaProvider; diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java index f414c92b8..1e5ca6886 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java @@ -30,6 +30,9 @@ import org.apache.hudi.integ.testsuite.dag.nodes.UpsertNode; import org.apache.hudi.integ.testsuite.dag.WorkflowDag; import org.junit.jupiter.api.Test; +/** + * Unit test for the build process of {@link DagNode} and {@link WorkflowDag}. + */ public class TestWorkflowBuilder { @Test diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java index d1c2b76fd..c48d1b13f 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java @@ -23,6 +23,7 @@ import static junit.framework.TestCase.assertTrue; import java.util.Arrays; import java.util.List; import java.util.Map; + import org.apache.avro.Schema.Field; import org.apache.avro.generic.GenericRecord; import org.apache.hudi.integ.testsuite.utils.TestUtils; @@ -34,6 +35,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import scala.Tuple2; +/** + * Test Cases for {@link UpdateConverter} APIs. + */ public class TestUpdateConverter { private JavaSparkContext jsc; @@ -49,11 +53,16 @@ public class TestUpdateConverter { jsc.stop(); } + /** + * Test {@link UpdateConverter} by generates random updates from existing records. + */ @Test public void testGenerateUpdateRecordsFromInputRecords() throws Exception { + // 1. prepare input records JavaRDD inputRDD = TestUtils.makeRDD(jsc, 10); String schemaStr = inputRDD.take(1).get(0).getSchema().toString(); int minPayloadSize = 1000; + // 2. DFS converter reads existing records and generates random updates for the same row keys UpdateConverter updateConverter = new UpdateConverter(schemaStr, minPayloadSize, Arrays.asList("timestamp"), Arrays.asList("_row_key")); diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java index a91ea012c..44b0ccd2f 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java @@ -29,15 +29,20 @@ import org.apache.hudi.integ.testsuite.dag.nodes.ValidateNode; import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config; import org.apache.spark.api.java.JavaRDD; +/** + * An implementation of {@link WorkflowDagGenerator}, that generates complex workflowDag. + */ public class ComplexDagGenerator implements WorkflowDagGenerator { @Override public WorkflowDag build() { + // root node DagNode root = new InsertNode(Config.newBuilder() .withNumRecordsToInsert(1000) .withNumInsertPartitions(3) .withRecordSize(1000).build()); + // child node1 DagNode child1 = new UpsertNode(Config.newBuilder() .withNumRecordsToUpdate(999) .withNumRecordsToInsert(1000) @@ -46,6 +51,7 @@ public class ComplexDagGenerator implements WorkflowDagGenerator { .withNumInsertPartitions(1) .withRecordSize(10000).build()); + // function used to build ValidateNode Function>>, Boolean> function = (dagNodes) -> { DagNode> parent1 = dagNodes.get(0); List statuses = parent1.getResult().collect(); @@ -63,8 +69,11 @@ public class ComplexDagGenerator implements WorkflowDagGenerator { * parent2.getConfig().getNumInsertPartitions() + parent2.getConfig().getNumRecordsUpsert(); return b1 & b2 & b3; }; + + // child node2 DagNode child2 = new ValidateNode(Config.newBuilder().build(), function); + // create relationship between nodes root.addChildNode(child1); // child1.addParentNode(root); child1.addChildNode(child2); diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java index 7309d430b..e2f6ee584 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java @@ -28,6 +28,9 @@ import org.apache.hudi.integ.testsuite.dag.nodes.InsertNode; import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config; import org.apache.hudi.integ.testsuite.dag.nodes.HiveQueryNode; +/** + * An implementation of {@link WorkflowDagGenerator}, helps to generate a workflowDag with two hive nodes as child node. + */ public class HiveSyncDagGenerator implements WorkflowDagGenerator { @Override diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java index 26269fcfa..07dd467a2 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java @@ -28,6 +28,10 @@ import org.apache.hudi.integ.testsuite.dag.nodes.InsertNode; import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config; import org.apache.hudi.integ.testsuite.dag.nodes.HiveQueryNode; +/** + * An implementation of {@link WorkflowDagGenerator}, that generates workflowDag with hive nodes as child node + * for MOR table. + */ public class HiveSyncDagGeneratorMOR implements WorkflowDagGenerator { @Override diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java index b5a0cd39f..418e64437 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java @@ -29,6 +29,9 @@ import org.apache.hudi.utilities.testutils.UtilitiesTestBase; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +/** + * A utility class for DAG test. + */ public class TestDagUtils { private static final String COW_DAG_DOCKER_DEMO_RELATIVE_PATH = "/docker/demo/config/test-suite/complex-dag-cow.yaml"; diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java index 85d535840..c31a7d673 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java @@ -25,6 +25,9 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link GenericRecordFullPayloadSizeEstimator}. + */ public class TestGenericRecordPayloadEstimator { private static final String SOURCE_SCHEMA_DOCKER_DEMO_RELATIVE_PATH = "/docker/demo/config/test-suite/source.avsc"; diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java index 886fb160c..7524d4af8 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java @@ -32,6 +32,9 @@ import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link GenericRecordFullPayloadGenerator} and {@link GenericRecordPartialPayloadGenerator}. + */ public class TestGenericRecordPayloadGenerator { private static final String SOURCE_SCHEMA_DOCKER_DEMO_RELATIVE_PATH = "/docker/demo/config/test-suite/source.avsc"; diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java index 4c71b6064..582601983 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java @@ -49,6 +49,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +/** + * Unit test against {@link HoodieTestSuiteJob}. + */ public class TestHoodieTestSuiteJob extends UtilitiesTestBase { private static final String TEST_NAME_WITH_PARAMS = "[{index}] Test with useDeltaStreamer={0}, tableType={1}"; diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java index 6d0b99493..fa8f4ac41 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java @@ -36,6 +36,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link DFSAvroDeltaInputReader} by issuing analyzeSingleFile and read from it. + */ public class TestDFSAvroDeltaInputReader extends UtilitiesTestBase { @BeforeAll diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java index 911f16891..941ef43ca 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java @@ -40,6 +40,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link DFSHoodieDatasetInputReader}. + */ public class TestDFSHoodieDatasetInputReader extends UtilitiesTestBase { @BeforeAll diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java index 4699324fd..372d6748c 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java @@ -28,6 +28,9 @@ import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; +/** + * A utility class for testing purpose. + */ public class TestUtils { /** @@ -45,6 +48,15 @@ public class TestUtils { return dataGenerator.generateGenericRecords(numRecords); } + /** + * Method helps to create avro files and save it to file. + * + * @param jsc Java Spark Context jsc. + * @param sparkSession Spark Session, the entry point to programming Spark with the Dataset and DataFrame API. + * @param basePath The basePath where files are written. + * @param numFiles The number of files to create. + * @param numRecordsPerFile The number of records per file. + */ public static void createAvroFiles(JavaSparkContext jsc, SparkSession sparkSession, String basePath, int numFiles, int numRecordsPerFile) { Schema schema = HoodieTestDataGenerator.AVRO_SCHEMA;