lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3272] If mode==ignore && tableExists, do not execute write logic and sync hive (#4632)

Browse Source
This commit is contained in:
董可伦
2022-02-14 11:52:00 +08:00
committed by GitHub
parent 93ee09fee8
commit 94806d5cf7
2 changed files with 45 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
View File

@@ -377,13 +377,15 @@ object HoodieSparkSqlWriter {
schema = HoodieAvroUtils.getNullSchema.toString schema = HoodieAvroUtils.getNullSchema.toString
} }
// Handle various save modes
if (mode == SaveMode.Ignore && tableExists) { if (mode == SaveMode.Ignore && tableExists) {
log.warn(s"hoodie table at $basePath already exists. Ignoring & not performing actual writes.") log.warn(s"hoodie table at $basePath already exists. Ignoring & not performing actual writes.")
if (!hoodieWriteClient.isEmpty) {
hoodieWriteClient.get.close()
}
false false
} else { } else {
// Handle various save modes
handleSaveModes(sqlContext.sparkSession, mode, basePath, tableConfig, tableName, WriteOperationType.BOOTSTRAP, fs) handleSaveModes(sqlContext.sparkSession, mode, basePath, tableConfig, tableName, WriteOperationType.BOOTSTRAP, fs)
}
if (!tableExists) { if (!tableExists) {
val archiveLogFolder = hoodieConfig.getStringOrDefault(HoodieTableConfig.ARCHIVELOG_FOLDER) val archiveLogFolder = hoodieConfig.getStringOrDefault(HoodieTableConfig.ARCHIVELOG_FOLDER)
@@ -423,6 +425,7 @@ object HoodieSparkSqlWriter {
val metaSyncSuccess = metaSync(sqlContext.sparkSession, hoodieConfig, basePath, df.schema) val metaSyncSuccess = metaSync(sqlContext.sparkSession, hoodieConfig, basePath, df.schema)
metaSyncSuccess metaSyncSuccess
} }
}
def bulkInsertAsRow(sqlContext: SQLContext, def bulkInsertAsRow(sqlContext: SQLContext,
parameters: Map[String, String], parameters: Map[String, String],

6
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
View File

@@ -543,6 +543,12 @@ class TestHoodieSparkSqlWriter {
// Verify that HoodieWriteClient is closed correctly // Verify that HoodieWriteClient is closed correctly
verify(client, times(1)).close() verify(client, times(1)).close()
val ignoreResult = HoodieSparkSqlWriter.bootstrap(sqlContext, SaveMode.Ignore, fooTableModifier, spark.emptyDataFrame, Option.empty,
Option(client))
assertFalse(ignoreResult)
verify(client, times(2)).close()
// fetch all records from parquet files generated from write to hudi // fetch all records from parquet files generated from write to hudi
val actualDf = sqlContext.read.parquet(tempBasePath) val actualDf = sqlContext.read.parquet(tempBasePath)
assert(actualDf.count == 100) assert(actualDf.count == 100)