lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-780] Migrate test cases to Junit 5 (#1504)

Browse Source
This commit is contained in:
Raymond Xu
2020-04-15 12:35:01 -07:00
committed by GitHub
parent 14d4fea833
commit d65efe659d
26 changed files with 522 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
View File

@@ -33,8 +33,7 @@ import com.github.dockerjava.core.command.ExecStartResultCallback;
import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Before;
import org.junit.jupiter.api.BeforeEach;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -46,6 +45,8 @@ import java.util.stream.Collectors;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
public abstract class ITTestBase {
@@ -113,12 +114,12 @@ public abstract class ITTestBase {
static String getPrestoConsoleCommand(String commandFile) {
StringBuilder builder = new StringBuilder().append("presto --server " + PRESTO_COORDINATOR_URL)
.append(" --catalog hive --schema default")
.append(" -f " + commandFile );
.append(" -f " + commandFile);
System.out.println("Presto comamnd " + builder.toString());
return builder.toString();
}
@Before
@BeforeEach
public void init() {
String dockerHost = (OVERRIDDEN_DOCKER_HOST != null) ? OVERRIDDEN_DOCKER_HOST : DEFAULT_DOCKER_HOST;
// Assuming insecure docker engine
@@ -165,10 +166,9 @@ public abstract class ITTestBase {
LOG.error("\n\n ###### Stderr #######\n" + callback.getStderr().toString());
if (expectedToSucceed) {
Assert.assertEquals("Command (" + Arrays.toString(command) + ") expected to succeed. Exit (" + exitCode + ")", 0, exitCode);
assertEquals(0, exitCode, "Command (" + Arrays.toString(command) + ") expected to succeed. Exit (" + exitCode + ")");
} else {
Assert.assertTrue("Command (" + Arrays.toString(command) + ") expected to fail. Exit (" + exitCode + ")",
exitCode != 0);
assertNotEquals(0, exitCode, "Command (" + Arrays.toString(command) + ") expected to fail. Exit (" + exitCode + ")");
}
cmd.close();
return callback;
@@ -224,7 +224,7 @@ public abstract class ITTestBase {
return Pair.of(callback.getStdout().toString().trim(), callback.getStderr().toString().trim());
}
void executePrestoCopyCommand(String fromFile, String remotePath){
void executePrestoCopyCommand(String fromFile, String remotePath) {
Container sparkWorkerContainer = runningContainers.get(PRESTO_COORDINATOR);
dockerClient.copyArchiveToContainerCmd(sparkWorkerContainer.getId())
.withHostResource(fromFile)
@@ -268,7 +268,7 @@ public abstract class ITTestBase {
saveUpLogs();
}
Assert.assertEquals("Did not find output the expected number of times", times, count);
assertEquals(times, count, "Did not find output the expected number of times");
}
public class TestExecStartResultCallback extends ExecStartResultCallback {

20
hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
View File

@@ -21,7 +21,7 @@ package org.apache.hudi.integ;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import java.util.Collections;
import java.util.List;
@@ -102,9 +102,9 @@ public class ITTestHoodieDemo extends ITTestBase {
private void setupDemo() throws Exception {
List<String> cmds = CollectionUtils.createImmutableList("hdfs dfsadmin -safemode wait",
"hdfs dfs -mkdir -p " + HDFS_DATA_DIR,
"hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH1 + " " + HDFS_BATCH_PATH1,
"/bin/bash " + DEMO_CONTAINER_SCRIPT);
"hdfs dfs -mkdir -p " + HDFS_DATA_DIR,
"hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH1 + " " + HDFS_BATCH_PATH1,
"/bin/bash " + DEMO_CONTAINER_SCRIPT);
executeCommandStringsInDocker(ADHOC_1_CONTAINER, cmds);
@@ -113,21 +113,21 @@ public class ITTestHoodieDemo extends ITTestBase {
executeCommandStringsInDocker(PRESTO_COORDINATOR, cmds);
// copy presto sql files to presto coordinator
executePrestoCopyCommand( System.getProperty("user.dir") + "/.." + PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR);
executePrestoCopyCommand( System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR);
executePrestoCopyCommand( System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR);
executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR);
executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR);
executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR);
}
private void ingestFirstBatchAndHiveSync() throws Exception {
List<String> cmds = CollectionUtils.createImmutableList(
"spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
"spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+ " --table-type COPY_ON_WRITE "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+ " --target-base-path " + COW_BASE_PATH + " --target-table " + COW_TABLE_NAME
+ " --props /var/demo/config/dfs-source.properties"
+ " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+ String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME),
("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+ " --table-type MERGE_ON_READ "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+ " --target-base-path " + MOR_BASE_PATH + " --target-table " + MOR_TABLE_NAME
@@ -286,7 +286,7 @@ public class ITTestHoodieDemo extends ITTestBase {
testIncrementalHiveQuery(MIN_COMMIT_TIME_COW_SCRIPT, HIVE_INCREMENTAL_COW_COMMANDS, expectedOutput, 1);
// verify that 10:59 is NOT present in RO table because of pending compaction
testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RO_COMMANDS, expectedOutput, 0);
testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RO_COMMANDS, expectedOutput, 0);
// verify that 10:59 is present in RT table even with pending compaction
testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RT_COMMANDS, expectedOutput, 1);

33
hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java
View File

@@ -18,12 +18,15 @@
package org.apache.hudi.integ;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.model.HoodieTableType;
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Smoke tests to run as part of verification.
@@ -129,7 +132,7 @@ public class ITTestHoodieSanity extends ITTestBase {
// Ensure table does not exist
Pair<String, String> stdOutErr = executeHiveCommand("show tables like '" + hiveTableName + "'");
Assert.assertTrue("Dropped table " + hiveTableName + " exists!", stdOutErr.getLeft().isEmpty());
assertTrue(stdOutErr.getLeft().isEmpty(), "Dropped table " + hiveTableName + " exists!");
// Run Hoodie Java App
String cmd;
@@ -145,24 +148,24 @@ public class ITTestHoodieSanity extends ITTestBase {
}
executeCommandStringInDocker(ADHOC_1_CONTAINER, cmd, true);
String snapshotTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name()) ?
hiveTableName + "_rt" : hiveTableName;
Option<String> roTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name()) ?
Option.of(hiveTableName +"_ro") : Option.empty();
String snapshotTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name())
? hiveTableName + "_rt" : hiveTableName;
Option<String> roTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name())
? Option.of(hiveTableName + "_ro") : Option.empty();
// Ensure table does exist
stdOutErr = executeHiveCommand("show tables like '" + snapshotTableName + "'");
Assert.assertEquals("Table exists", snapshotTableName, stdOutErr.getLeft());
assertEquals(snapshotTableName, stdOutErr.getLeft(), "Table exists");
// Ensure row count is 80 (without duplicates) (100 - 20 deleted)
stdOutErr = executeHiveCommand("select count(1) from " + snapshotTableName);
Assert.assertEquals("Expecting 80 rows to be present in the snapshot table", 80,
Integer.parseInt(stdOutErr.getLeft().trim()));
assertEquals(80, Integer.parseInt(stdOutErr.getLeft().trim()),
"Expecting 80 rows to be present in the snapshot table");
if (roTableName.isPresent()) {
stdOutErr = executeHiveCommand("select count(1) from " + roTableName.get());
Assert.assertEquals("Expecting 80 rows to be present in the snapshot table", 80,
Integer.parseInt(stdOutErr.getLeft().trim()));
assertEquals(80, Integer.parseInt(stdOutErr.getLeft().trim()),
"Expecting 80 rows to be present in the snapshot table");
}
// Make the HDFS dataset non-hoodie and run the same query; Checks for interoperability with non-hoodie tables
@@ -175,8 +178,8 @@ public class ITTestHoodieSanity extends ITTestBase {
} else {
stdOutErr = executeHiveCommand("select count(1) from " + snapshotTableName);
}
Assert.assertEquals("Expecting 280 rows to be present in the new table", 280,
Integer.parseInt(stdOutErr.getLeft().trim()));
assertEquals(280, Integer.parseInt(stdOutErr.getLeft().trim()),
"Expecting 280 rows to be present in the new table");
}
private void dropHiveTables(String hiveTableName, String tableType) throws Exception {