lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1635] Improvements to Hudi Test Suite (#2628)

Browse Source
This commit is contained in:
Balajee Nagasubramaniam
2021-03-09 13:29:38 -08:00
committed by GitHub
parent d3a451611c
commit d8af24d8a2
13 changed files with 360 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
View File

@@ -47,7 +47,9 @@ public class HiveSyncDagGeneratorMOR implements WorkflowDagGenerator {
root.addChildNode(child1);
DagNode child2 = new HiveQueryNode(Config.newBuilder().withHiveLocal(true).withHiveQueryAndResults(Arrays
.asList(Pair.of("select " + "count(*) from testdb1.table1_rt group " + "by rider having count(*) < 1", 0)))
.asList(Pair.of("select " + "count(*) from testdb1.hive_trips group " + "by rider having count(*) < 1", 0),
Pair.of("select " + "count(*) from testdb1.hive_trips ", 100)))
.withHiveProperties(Arrays.asList("set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat"))
.build());
child1.addChildNode(child2);

34
hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
View File

@@ -43,6 +43,40 @@ public class TestDagUtils {
System.out.println(yaml);
}
@Test
public void testConvertDagToYamlHiveQuery() throws Exception {
WorkflowDag dag = new HiveSyncDagGenerator().build();
DagNode insert1 = (DagNode) dag.getNodeList().get(0);
DagNode hiveSync1 = (DagNode)insert1.getChildNodes().get(0);
DagNode hiveQuery1 = (DagNode)hiveSync1.getChildNodes().get(0);
String yaml = DagUtils.convertDagToYaml(dag);
WorkflowDag dag2 = DagUtils.convertYamlToDag(yaml);
DagNode insert2 = (DagNode) dag2.getNodeList().get(0);
DagNode hiveSync2 = (DagNode)insert2.getChildNodes().get(0);
DagNode hiveQuery2 = (DagNode)hiveSync2.getChildNodes().get(0);
assertEquals(hiveQuery1.getConfig().getHiveQueries().get(0),
hiveQuery2.getConfig().getHiveQueries().get(0));
assertEquals(hiveQuery1.getConfig().getHiveProperties().get(0),
hiveQuery2.getConfig().getHiveProperties().get(0));
}
@Test
public void testConvertDagToYamlAndBack() throws Exception {
final ComplexDagGenerator dag = new ComplexDagGenerator();
final WorkflowDag originalWorkflowDag = dag.build();
final String yaml = DagUtils.convertDagToYaml(dag.build());
final WorkflowDag regeneratedWorkflowDag = DagUtils.convertYamlToDag(yaml);
final List<DagNode> originalWorkflowDagNodes = originalWorkflowDag.getNodeList();
final List<DagNode> regeneratedWorkflowDagNodes = regeneratedWorkflowDag.getNodeList();
assertEquals(originalWorkflowDagNodes.size(), regeneratedWorkflowDagNodes.size());
assertEquals(originalWorkflowDagNodes.get(0).getChildNodes().size(),
regeneratedWorkflowDagNodes.get(0).getChildNodes().size());
}
@Test
public void testConvertYamlToDag() throws Exception {
WorkflowDag dag = DagUtils.convertYamlToDag(UtilitiesTestBase.Helpers

8
hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
View File

@@ -144,9 +144,8 @@ public class TestGenericRecordPayloadGenerator {
List<Long> insertTimeStamps = new ArrayList<>();
List<Long> updateTimeStamps = new ArrayList<>();
List<GenericRecord> records = new ArrayList<>();
Long startMillis = System.currentTimeMillis() - TimeUnit.MILLISECONDS
.convert(GenericRecordFullPayloadGenerator.DEFAULT_NUM_DATE_PARTITIONS, TimeUnit.DAYS);
Long startSeconds = 0L;
Long endSeconds = TimeUnit.SECONDS.convert(10, TimeUnit.DAYS);
// Generate 10 new records
IntStream.range(0, 10).forEach(a -> {
GenericRecord record = payloadGenerator.getNewPayloadWithTimestamp("timestamp");
@@ -165,7 +164,6 @@ public class TestGenericRecordPayloadGenerator {
assertTrue(insertRowKeys.containsAll(updateRowKeys));
// The timestamp field for the insert payloads should not all match with the update payloads
assertFalse(insertTimeStamps.containsAll(updateTimeStamps));
Long currentMillis = System.currentTimeMillis();
assertTrue(insertTimeStamps.stream().allMatch(t -> t >= startMillis && t <= currentMillis));
assertTrue(insertTimeStamps.stream().allMatch(t -> t >= startSeconds && t <= endSeconds));
}
}