lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3060] drop table for spark sql (#4364)

Browse Source
This commit is contained in:
ForwardXu
2021-12-22 19:17:43 +08:00
committed by GitHub
parent 1a5f8693aa
commit 5d93edc539
4 changed files with 203 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDropTable.scala Normal file
View File

@@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.sql.hudi
class TestDropTable extends TestHoodieSqlBase {
test("Test Drop Table") {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
// create table
spark.sql(
s"""
|create table $tableName (
| id int,
| name string,
| price double,
| ts long
|) using hudi
| location '${tmp.getCanonicalPath}/$tableName'
| tblproperties (
| type = '$tableType',
| primaryKey = 'id',
| preCombineField = 'ts'
| )
""".stripMargin)
spark.sql(s"DROP TABLE $tableName")
checkAnswer(s"show tables like '$tableName'")()
assertResult(true)(existsPath(s"${tmp.getCanonicalPath}/$tableName"))
}
}
}
test("Test Drop Table with purge") {
withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
val tableName = generateTableName
// create table
spark.sql(
s"""
|create table $tableName (
| id int,
| name string,
| price double,
| ts long
|) using hudi
| location '${tmp.getCanonicalPath}/$tableName'
| tblproperties (
| type = '$tableType',
| primaryKey = 'id',
| preCombineField = 'ts'
| )
""".stripMargin)
spark.sql(s"DROP TABLE $tableName PURGE")
checkAnswer(s"show tables like '$tableName'")()
assertResult(false)(existsPath(s"${tmp.getCanonicalPath}/$tableName"))
}
}
}
}

10
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieSqlBase.scala
View File

@@ -17,7 +17,8 @@
package org.apache.spark.sql.hudi
import java.io.File
import org.apache.hadoop.fs.Path
import org.apache.hudi.common.fs.FSUtils
import org.apache.log4j.Level
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.{Row, SparkSession}
@@ -25,6 +26,7 @@ import org.apache.spark.util.Utils
import org.scalactic.source
import org.scalatest.{BeforeAndAfterAll, FunSuite, Tag}
import java.io.File
import java.util.TimeZone
class TestHoodieSqlBase extends FunSuite with BeforeAndAfterAll {
@@ -115,4 +117,10 @@ class TestHoodieSqlBase extends FunSuite with BeforeAndAfterAll {
case _=> value
}
}
protected def existsPath(filePath: String): Boolean = {
val path = new Path(filePath)
val fs = FSUtils.getFs(filePath, spark.sparkContext.hadoopConfiguration)
fs.exists(path)
}
}