lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2077] Fix TestHoodieDeltaStreamerWithMultiWriter (#3849)

Browse Source 
Remove the logic of using deltastreamer to prep test table. Use fixture (compressed test table) instead.
This commit is contained in:
Raymond Xu
2021-10-24 21:14:39 -07:00
committed by GitHub
parent 91845e241d
commit d8560377c3
10 changed files with 178 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
View File

@@ -176,8 +176,14 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
}
}
/**
* To clean up Spark resources after all testcases have run in functional tests.
*
* Spark session and contexts were reused for testcases in the same test class. Some
* testcase may invoke this specifically to clean up in case of repeated test runs.
*/
@AfterAll
public static synchronized void cleanUpAfterAll() {
public static synchronized void resetSpark() {
if (spark != null) {
spark.close();
spark = null;

4
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/providers/SparkProvider.java
View File

@@ -39,6 +39,8 @@ public interface SparkProvider extends org.apache.hudi.testutils.providers.Hoodi
SparkConf sparkConf = new SparkConf();
sparkConf.set("spark.app.name", getClass().getName());
sparkConf.set("spark.master", "local[*]");
sparkConf.set("spark.default.parallelism", "4");
sparkConf.set("spark.sql.shuffle.partitions", "4");
sparkConf.set("spark.driver.maxResultSize", "2g");
sparkConf.set("spark.hadoop.mapred.output.compress", "true");
sparkConf.set("spark.hadoop.mapred.output.compression.codec", "true");
@@ -52,4 +54,4 @@ public interface SparkProvider extends org.apache.hudi.testutils.providers.Hoodi
default SparkConf conf() {
return conf(Collections.emptyMap());
}
}
}