lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4168] Add Call Procedure for marker deletion (#5738)

Browse Source 
* Add Call Procedure for marker deletion
This commit is contained in:
Saisai Shao
2022-06-05 11:05:38 +08:00
committed by GitHub
parent 80783c27f5
commit bd26d633d7
3 changed files with 119 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallProcedure.scala
View File

@@ -17,6 +17,8 @@
package org.apache.spark.sql.hudi.procedure
import org.apache.hudi.common.model.IOType
import org.apache.hudi.common.testutils.FileCreateUtils
import org.apache.spark.sql.hudi.HoodieSparkSqlTestBase
class TestCallProcedure extends HoodieSparkSqlTestBase {
@@ -131,4 +133,42 @@ class TestCallProcedure extends HoodieSparkSqlTestBase {
assertResult(1){commits.length}
}
}
test("Test Call delete_marker Procedure") {
withTempDir { tmp =>
val tableName = generateTableName
val tablePath = s"${tmp.getCanonicalPath}/$tableName"
// create table
spark.sql(
s"""
|create table $tableName (
| id int,
| name string,
| price double,
| ts long
|) using hudi
| location '$tablePath'
| tblproperties (
| primaryKey = 'id',
| preCombineField = 'ts'
| )
""".stripMargin)
// Check required fields
checkExceptionContain(s"""call delete_marker(table => '$tableName')""")(
s"Argument: instant_Time is required")
val instantTime = "101"
FileCreateUtils.createMarkerFile(tablePath, "", instantTime, "f0", IOType.APPEND)
assertResult(1) {
FileCreateUtils.getTotalMarkerFileCount(tablePath, "", instantTime, IOType.APPEND)
}
checkAnswer(s"""call delete_marker(table => '$tableName', instant_Time => '$instantTime')""")(Seq(true))
assertResult(0) {
FileCreateUtils.getTotalMarkerFileCount(tablePath, "", instantTime, IOType.APPEND)
}
}
}
}