lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2279]Support column name matching for insert * and update set * in merge into (#3415)

Browse Source
This commit is contained in:
董可伦
2021-08-13 14:10:07 +08:00
committed by GitHub
parent 0544d70d8f
commit 6602e55cd2
2 changed files with 102 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala
View File

@@ -161,11 +161,25 @@ case class HoodieResolveReferences(sparkSession: SparkSession) extends Rule[Logi
val resolvedCondition = condition.map(resolveExpressionFrom(resolvedSource)(_))
val resolvedAssignments = if (isInsertOrUpdateStar(assignments)) {
// assignments is empty means insert * or update set *
// we fill assign all the source fields to the target fields
target.output
.filter(attr => !HoodieSqlUtils.isMetaField(attr.name))
.zip(resolvedSource.output.filter(attr => !HoodieSqlUtils.isMetaField(attr.name)))
.map { case (targetAttr, sourceAttr) => Assignment(targetAttr, sourceAttr) }
val resolvedSourceOutputWithoutMetaFields = resolvedSource.output.filter(attr => !HoodieSqlUtils.isMetaField(attr.name))
val targetOutputWithoutMetaFields = target.output.filter(attr => !HoodieSqlUtils.isMetaField(attr.name))
val resolvedSourceColumnNamesWithoutMetaFields = resolvedSourceOutputWithoutMetaFields.map(_.name)
val targetColumnNamesWithoutMetaFields = targetOutputWithoutMetaFields.map(_.name)
if(targetColumnNamesWithoutMetaFields.toSet.subsetOf(resolvedSourceColumnNamesWithoutMetaFields.toSet)){
//If sourceTable's columns contains all targetTable's columns,
//We fill assign all the source fields to the target fields by column name matching.
val sourceColNameAttrMap = resolvedSourceOutputWithoutMetaFields.map(attr => (attr.name, attr)).toMap
targetOutputWithoutMetaFields.map(targetAttr => {
val sourceAttr = sourceColNameAttrMap(targetAttr.name)
Assignment(targetAttr, sourceAttr)
})
} else {
// We fill assign all the source fields to the target fields by order.
targetOutputWithoutMetaFields
.zip(resolvedSourceOutputWithoutMetaFields)
.map { case (targetAttr, sourceAttr) => Assignment(targetAttr, sourceAttr) }
}
} else {
assignments.map(assignment => {
val resolvedKey = resolveExpressionFrom(target)(assignment.key)