lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-372] Support the shortName for Hudi DataSource (#1054)

Browse Source 
- Ability to do `spark.write.format("hudi")...`
This commit is contained in:
lamber-ken
2019-12-01 00:02:33 +08:00
committed by vinoth chandar
parent 0b52ae3ac2
commit 024230fbd2
3 changed files with 34 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
hudi-spark/src/test/scala/TestDataSource.scala
View File

@@ -63,6 +63,19 @@ class TestDataSource extends AssertionsForJUnit {
fs = FSUtils.getFs(basePath, spark.sparkContext.hadoopConfiguration)
}
@Test def testShortNameStorage() {
// Insert Operation
val records = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("000", 100)).toList
val inputDF: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records, 2))
inputDF.write.format("hudi")
.options(commonOpts)
.option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL)
.mode(SaveMode.Overwrite)
.save(basePath)
assertTrue(HoodieDataSourceHelpers.hasNewCommits(fs, basePath, "000"))
}
@Test def testCopyOnWriteStorage() {
// Insert Operation
val records1 = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("000", 100)).toList