lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-995] Use Transformations, Assertions and SchemaTestUtil (#1884)

Browse Source 
- Consolidate transform functions for tests in Transformations.java
- Consolidate assertion functions for tests in Assertions.java
- Make use of SchemaTestUtil for loading schema from resource
This commit is contained in:
Raymond Xu
2020-08-01 05:57:18 -07:00
committed by GitHub
parent e79fbc07fe
commit 10e4268792
23 changed files with 302 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-spark/src/test/java/HoodieJavaStreamingApp.java
View File

@@ -23,7 +23,6 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hive.MultiPartKeysValueExtractor;
import org.apache.hudi.testutils.DataSourceTestUtils;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -45,6 +44,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import static org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings;
/**
* Sample program that writes & reads hoodie tables via the Spark datasource streaming.
*/
@@ -128,10 +129,10 @@ public class HoodieJavaStreamingApp {
// Generator of some records to be loaded in.
HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
List<String> records1 = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("001", 100));
List<String> records1 = recordsToStrings(dataGen.generateInserts("001", 100));
Dataset<Row> inputDF1 = spark.read().json(jssc.parallelize(records1, 2));
List<String> records2 = DataSourceTestUtils.convertToStringList(dataGen.generateUpdates("002", 100));
List<String> records2 = recordsToStrings(dataGen.generateUpdates("002", 100));
Dataset<Row> inputDF2 = spark.read().json(jssc.parallelize(records2, 2));