lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2339] Create Table If Not Exists Failed After Alter Table (#3510)

Browse Source
This commit is contained in:
pengzhiwei
2021-08-20 14:21:10 +08:00
committed by GitHub
parent c350d05dd3
commit 49829f8822
3 changed files with 30 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala
View File

@@ -77,7 +77,7 @@ object HoodieSqlUtils extends SparkAdapterSupport {
case _: Throwable => None
}
avroSchema.map(SchemaConverters.toSqlType(_).dataType
.asInstanceOf[StructType])
.asInstanceOf[StructType]).map(removeMetaFields)
}
private def tripAlias(plan: LogicalPlan): LogicalPlan = {

6
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala
View File

@@ -305,12 +305,14 @@ case class HoodieResolveReferences(sparkSession: SparkSession) extends Rule[Logi
case c @ CreateTable(tableDesc, _, _)
if isHoodieTable(tableDesc) =>
val tablePath = getTableLocation(c.tableDesc, sparkSession)
if (tableExistsInPath(tablePath, sparkSession.sessionState.newHadoopConf())) {
val tableExistInCatalog = sparkSession.sessionState.catalog.tableExists(tableDesc.identifier)
// Only when the table has not exist in catalog, we need to fill the schema info for creating table.
if (!tableExistInCatalog && tableExistsInPath(tablePath, sparkSession.sessionState.newHadoopConf())) {
val metaClient = HoodieTableMetaClient.builder()
.setBasePath(tablePath)
.setConf(sparkSession.sessionState.newHadoopConf())
.build()
val tableSchema = HoodieSqlUtils.getTableSqlSchema(metaClient).map(HoodieSqlUtils.addMetaFields)
val tableSchema = HoodieSqlUtils.getTableSqlSchema(metaClient)
if (tableSchema.isDefined && tableDesc.schema.isEmpty) {
// Fill the schema with the schema from the table
c.copy(tableDesc.copy(schema = tableSchema.get))

25
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
View File

@@ -505,4 +505,29 @@ class TestCreateTable extends TestHoodieSqlBase {
}
}
test("Test Create Table Exists In Catalog") {
val tableName = generateTableName
spark.sql(
s"""
|create table $tableName (
| id int,
| name string,
| price double
|) using hudi
|""".stripMargin
)
spark.sql(s"alter table $tableName add columns(ts bigint)")
// Check "create table if not exist" works after schema evolution.
spark.sql(
s"""
|create table if not exists $tableName (
| id int,
| name string,
| price double
|) using hudi
|""".stripMargin
)
}
}