lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3889] Do not validate table config if save mode is set to Overwrite (#5619)

Browse Source 
Co-authored-by: xicm <xicm@asiainfo.com>
This commit is contained in:
xi chaomin
2022-06-10 07:23:51 +08:00
committed by GitHub
parent ba47904fa2
commit 2b3a85528a
3 changed files with 65 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
View File

@@ -272,6 +272,29 @@ class TestHoodieSparkSqlWriter {
assert(tableAlreadyExistException.getMessage.contains(s"${HoodieWriteConfig.TBL_NAME.key}:\thoodie_bar_tbl\thoodie_foo_tbl"))
}
/**
* Test case for Do not validate table config if save mode is set to Overwrite
*/
@Test
def testValidateTableConfigWithOverwriteSaveMode(): Unit = {
//create a new table
val tableModifier1 = Map("path" -> tempBasePath, HoodieWriteConfig.TBL_NAME.key -> hoodieFooTableName,
"hoodie.datasource.write.recordkey.field" -> "uuid")
val dataFrame = spark.createDataFrame(Seq(StringLongTest(UUID.randomUUID().toString, new Date().getTime)))
HoodieSparkSqlWriter.write(sqlContext, SaveMode.Overwrite, tableModifier1, dataFrame)
//on same path try write with different RECORDKEY_FIELD_NAME and Append SaveMode should throw an exception
val tableModifier2 = Map("path" -> tempBasePath, HoodieWriteConfig.TBL_NAME.key -> hoodieFooTableName,
"hoodie.datasource.write.recordkey.field" -> "ts")
val dataFrame2 = spark.createDataFrame(Seq(StringLongTest(UUID.randomUUID().toString, new Date().getTime)))
val hoodieException = intercept[HoodieException](HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, tableModifier2, dataFrame2))
assert(hoodieException.getMessage.contains("Config conflict"))
assert(hoodieException.getMessage.contains(s"RecordKey:\tts\tuuid"))
//on same path try write with different RECORDKEY_FIELD_NAME and Overwrite SaveMode should be successful.
assert(HoodieSparkSqlWriter.write(sqlContext, SaveMode.Overwrite, tableModifier2, dataFrame2)._1)
}
/**
* Test case for each bulk insert sort mode
*