lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3722] Fix truncate hudi table's error (#5140)

Browse Source
This commit is contained in:
ForwardXu
2022-03-29 09:44:18 +08:00
committed by GitHub
parent d074089c62
commit 72e0b52b18
5 changed files with 232 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestAlterTableDropPartition.scala
View File

@@ -224,7 +224,7 @@ class TestAlterTableDropPartition extends TestHoodieSqlBase {
// not specified all partition column
checkExceptionContain(s"alter table $tableName drop partition (year='2021', month='10')")(
"All partition columns need to be specified for Hoodie's dropping partition"
"All partition columns need to be specified for Hoodie's partition"
)
// drop 2021-10-01 partition
spark.sql(s"alter table $tableName drop partition (year='2021', month='10', day='01')")

98
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTruncateTable.scala
View File

@@ -18,9 +18,14 @@
package org.apache.spark.sql.hudi
import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.keygen.{ComplexKeyGenerator, SimpleKeyGenerator}
import org.apache.spark.sql.SaveMode
class TestTruncateTable extends TestHoodieSqlBase {
test("Test Truncate Table") {
test("Test Truncate non-partitioned Table") {
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
// Create table
@@ -51,4 +56,95 @@ class TestTruncateTable extends TestHoodieSqlBase {
)
}
}
Seq(false, true).foreach { urlencode =>
test(s"Test Truncate single-partition table' partitions, urlencode: $urlencode") {
withTempDir { tmp =>
val tableName = generateTableName
val tablePath = s"${tmp.getCanonicalPath}/$tableName"
import spark.implicits._
val df = Seq((1, "z3", "v1", "2021/10/01"), (2, "l4", "v1", "2021/10/02"))
.toDF("id", "name", "ts", "dt")
df.write.format("hudi")
.option(HoodieWriteConfig.TBL_NAME.key, tableName)
.option(TABLE_TYPE.key, MOR_TABLE_TYPE_OPT_VAL)
.option(RECORDKEY_FIELD.key, "id")
.option(PRECOMBINE_FIELD.key, "ts")
.option(PARTITIONPATH_FIELD.key, "dt")
.option(URL_ENCODE_PARTITIONING.key(), urlencode)
.option(KEYGENERATOR_CLASS_NAME.key, classOf[SimpleKeyGenerator].getName)
.option(HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key, "1")
.option(HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key, "1")
.mode(SaveMode.Overwrite)
.save(tablePath)
// register meta to spark catalog by creating table
spark.sql(
s"""
|create table $tableName using hudi
|location '$tablePath'
|""".stripMargin)
// truncate 2021-10-01 partition
spark.sql(s"truncate table $tableName partition (dt='2021/10/01')")
checkAnswer(s"select dt from $tableName")(Seq(s"2021/10/02"))
// Truncate table
spark.sql(s"truncate table $tableName")
checkAnswer(s"select count(1) from $tableName")(Seq(0))
}
}
}
Seq(false, true).foreach { hiveStyle =>
test(s"Test Truncate multi-level partitioned table's partitions, isHiveStylePartitioning: $hiveStyle") {
withTempDir { tmp =>
val tableName = generateTableName
val tablePath = s"${tmp.getCanonicalPath}/$tableName"
import spark.implicits._
val df = Seq((1, "z3", "v1", "2021", "10", "01"), (2, "l4", "v1", "2021", "10","02"))
.toDF("id", "name", "ts", "year", "month", "day")
df.write.format("hudi")
.option(HoodieWriteConfig.TBL_NAME.key, tableName)
.option(TABLE_TYPE.key, COW_TABLE_TYPE_OPT_VAL)
.option(RECORDKEY_FIELD.key, "id")
.option(PRECOMBINE_FIELD.key, "ts")
.option(PARTITIONPATH_FIELD.key, "year,month,day")
.option(HIVE_STYLE_PARTITIONING.key, hiveStyle)
.option(KEYGENERATOR_CLASS_NAME.key, classOf[ComplexKeyGenerator].getName)
.option(HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key, "1")
.option(HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key, "1")
.mode(SaveMode.Overwrite)
.save(tablePath)
// register meta to spark catalog by creating table
spark.sql(
s"""
|create table $tableName using hudi
|location '$tablePath'
|""".stripMargin)
// not specified all partition column
checkExceptionContain(s"truncate table $tableName partition (year='2021', month='10')")(
"All partition columns need to be specified for Hoodie's partition"
)
// truncate 2021-10-01 partition
spark.sql(s"truncate table $tableName partition (year='2021', month='10', day='01')")
checkAnswer(s"select id, name, ts, year, month, day from $tableName")(
Seq(2, "l4", "v1", "2021", "10", "02")
)
// Truncate table
spark.sql(s"truncate table $tableName")
checkAnswer(s"select count(1) from $tableName")(Seq(0))
}
}
}
}