From a398aad1fcbd5ff36eb7bc750fcc418d78b3a97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=91=A3=E5=8F=AF=E4=BC=A6?= Date: Tue, 30 Nov 2021 14:36:36 +0800 Subject: [PATCH] [HUDI-2642] Add support ignoring case in update sql operation (#3882) --- .../command/UpdateHoodieTableCommand.scala | 5 +- .../spark/sql/hudi/TestUpdateTable.scala | 55 ++++++++++++++++--- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala index 60122cede..7397b0dad 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala @@ -51,7 +51,10 @@ case class UpdateHoodieTableCommand(updateTable: UpdateTable) extends RunnableCo }.toMap val updateExpressions = table.output - .map(attr => name2UpdateValue.getOrElse(attr.name, attr)) + .map(attr => { + val UpdateValueOption = name2UpdateValue.find(f => sparkSession.sessionState.conf.resolver(f._1, attr.name)) + if(UpdateValueOption.isEmpty) attr else UpdateValueOption.get._2 + }) .filter { // filter the meta columns case attr: AttributeReference => !HoodieRecord.HOODIE_META_COLUMNS.asScala.toSet.contains(attr.name) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala index 2537c9c80..82d067cfb 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestUpdateTable.scala @@ -27,16 +27,16 @@ class TestUpdateTable extends TestHoodieSqlBase { spark.sql( s""" |create table $tableName ( - | id int, - | name string, - | price double, - | ts long + | ID int, + | NAME string, + | PRICE double, + | TS long |) using hudi | location '${tmp.getCanonicalPath}/$tableName' - | tblproperties ( + | options ( | type = '$tableType', - | primaryKey = 'id', - | preCombineField = 'ts' + | primaryKey = 'ID', + | preCombineField = 'TS' | ) """.stripMargin) // insert data to table @@ -59,4 +59,45 @@ class TestUpdateTable extends TestHoodieSqlBase { } } } + + test("Test ignoring case for Update Table") { + withTempDir { tmp => + Seq("cow", "mor").foreach {tableType => + val tableName = generateTableName + // create table + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + |) using hudi + | location '${tmp.getCanonicalPath}/$tableName' + | tblproperties ( + | type = '$tableType', + | primaryKey = 'id', + | preCombineField = 'ts' + | ) + """.stripMargin) + // insert data to table + spark.sql(s"insert into $tableName select 1, 'a1', 10, 1000") + checkAnswer(s"select id, name, price, ts from $tableName")( + Seq(1, "a1", 10.0, 1000) + ) + + // update data + spark.sql(s"update $tableName set PRICE = 20 where ID = 1") + checkAnswer(s"select id, name, price, ts from $tableName")( + Seq(1, "a1", 20.0, 1000) + ) + + // update data + spark.sql(s"update $tableName set PRICE = PRICE * 2 where ID = 1") + checkAnswer(s"select id, name, price, ts from $tableName")( + Seq(1, "a1", 40.0, 1000) + ) + } + } + } }