lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

HUDI-528 Handle empty commit in incremental pulling (#1612)

Browse Source
This commit is contained in:
Gary Li
2020-05-14 22:55:25 -07:00
committed by GitHub
parent f094f42857
commit a64afdfd17
2 changed files with 20 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala
View File

@@ -126,6 +126,14 @@ class TestDataSource {
assertEquals(1, countsPerCommit.length)
assertEquals(firstCommit, countsPerCommit(0).get(0))
// Upsert an empty dataFrame
val emptyRecords = DataSourceTestUtils.convertToStringList(dataGen.generateUpdates("002", 0)).toList
val emptyDF: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(emptyRecords, 1))
emptyDF.write.format("org.apache.hudi")
.options(commonOpts)
.mode(SaveMode.Append)
.save(basePath)
// pull the latest commit
val hoodieIncViewDF2 = spark.read.format("org.apache.hudi")
.option(DataSourceReadOptions.QUERY_TYPE_OPT_KEY, DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL)