lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2642] Add support ignoring case in update sql operation (#3882)

Browse Source
This commit is contained in:
董可伦
2021-11-30 14:36:36 +08:00
committed by GitHub
parent 3433f00cb5
commit a398aad1fc
2 changed files with 52 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala
View File

@@ -51,7 +51,10 @@ case class UpdateHoodieTableCommand(updateTable: UpdateTable) extends RunnableCo
}.toMap
val updateExpressions = table.output
.map(attr => name2UpdateValue.getOrElse(attr.name, attr))
.map(attr => {
val UpdateValueOption = name2UpdateValue.find(f => sparkSession.sessionState.conf.resolver(f._1, attr.name))
if(UpdateValueOption.isEmpty) attr else UpdateValueOption.get._2
})
.filter { // filter the meta columns
case attr: AttributeReference =>
!HoodieRecord.HOODIE_META_COLUMNS.asScala.toSet.contains(attr.name)

55
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala
View File

@@ -27,16 +27,16 @@ class TestUpdateTable extends TestHoodieSqlBase {
spark.sql(
s"""
|create table $tableName (
| id int,
| name string,
| price double,
| ts long
| ID int,
| NAME string,
| PRICE double,
| TS long
|) using hudi
| location '${tmp.getCanonicalPath}/$tableName'
| tblproperties (
| options (
| type = '$tableType',
| primaryKey = 'id',
| preCombineField = 'ts'
| primaryKey = 'ID',
| preCombineField = 'TS'
| )
""".stripMargin)
// insert data to table
@@ -59,4 +59,45 @@ class TestUpdateTable extends TestHoodieSqlBase {
}
}
}
test("Test ignoring case for Update Table") {
withTempDir { tmp =>
Seq("cow", "mor").foreach {tableType =>
val tableName = generateTableName
// create table
spark.sql(
s"""
|create table $tableName (
| id int,
| name string,
| price double,
| ts long
|) using hudi
| location '${tmp.getCanonicalPath}/$tableName'
| tblproperties (
| type = '$tableType',
| primaryKey = 'id',
| preCombineField = 'ts'
| )
""".stripMargin)
// insert data to table
spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000")
checkAnswer(s"select id, name, price, ts from $tableName")(
Seq(1, "a1", 10.0, 1000)
)
// update data
spark.sql(s"update $tableName set PRICE = 20 where ID = 1")
checkAnswer(s"select id, name, price, ts from $tableName")(
Seq(1, "a1", 20.0, 1000)
)
// update data
spark.sql(s"update $tableName set PRICE = PRICE * 2 where ID = 1")
checkAnswer(s"select id, name, price, ts from $tableName")(
Seq(1, "a1", 40.0, 1000)
)
}
}
}
}