lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3236] use fields'comments persisted in catalog to fill in schema (#4587)

Browse Source
This commit is contained in:
Yann Byron
2022-01-20 13:44:35 +08:00
committed by GitHub
parent a08a2b7306
commit 31b57a256f
6 changed files with 75 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTable.scala
View File

@@ -18,6 +18,7 @@
package org.apache.spark.sql.hudi
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.types.{LongType, StructField, StructType}
@@ -44,7 +45,24 @@ class TestAlterTable extends TestHoodieSqlBase {
| preCombineField = 'ts'
| )
""".stripMargin)
// Alter table name.
// change column comment
spark.sql(s"alter table $tableName change column id id int comment 'primary id'")
var catalogTable = spark.sessionState.catalog.getTableMetadata(new TableIdentifier(tableName))
assertResult("primary id") (
catalogTable.schema(catalogTable.schema.fieldIndex("id")).getComment().get
)
spark.sql(s"alter table $tableName change column name name string comment 'name column'")
spark.sessionState.catalog.refreshTable(new TableIdentifier(tableName))
catalogTable = spark.sessionState.catalog.getTableMetadata(new TableIdentifier(tableName))
assertResult("primary id") (
catalogTable.schema(catalogTable.schema.fieldIndex("id")).getComment().get
)
assertResult("name column") (
catalogTable.schema(catalogTable.schema.fieldIndex("name")).getComment().get
)
// alter table name.
val newTableName = s"${tableName}_1"
spark.sql(s"alter table $tableName rename to $newTableName")
assertResult(false)(
@@ -53,24 +71,26 @@ class TestAlterTable extends TestHoodieSqlBase {
assertResult(true) (
spark.sessionState.catalog.tableExists(new TableIdentifier(newTableName))
)
val hadoopConf = spark.sessionState.newHadoopConf()
val metaClient = HoodieTableMetaClient.builder().setBasePath(tablePath)
.setConf(hadoopConf).build()
assertResult(newTableName) (
metaClient.getTableConfig.getTableName
)
assertResult(newTableName) (metaClient.getTableConfig.getTableName)
// insert some data
spark.sql(s"insert into $newTableName values(1, 'a1', 10, 1000)")
// Add table column
// add column
spark.sql(s"alter table $newTableName add columns(ext0 string)")
val table = spark.sessionState.catalog.getTableMetadata(new TableIdentifier(newTableName))
catalogTable = spark.sessionState.catalog.getTableMetadata(new TableIdentifier(newTableName))
assertResult(Seq("id", "name", "price", "ts", "ext0")) {
HoodieSqlCommonUtils.removeMetaFields(table.schema).fields.map(_.name)
HoodieSqlCommonUtils.removeMetaFields(catalogTable.schema).fields.map(_.name)
}
checkAnswer(s"select id, name, price, ts, ext0 from $newTableName")(
Seq(1, "a1", 10.0, 1000, null)
)
// Alter table column type
// change column's data type
spark.sql(s"alter table $newTableName change column id id bigint")
assertResult(StructType(Seq(StructField("id", LongType, nullable = true))))(
spark.sql(s"select id from $newTableName").schema)