From 0802510ca9d55554e87e9aee9cbf9e694dd8a785 Mon Sep 17 00:00:00 2001 From: ForwardXu Date: Wed, 30 Mar 2022 02:28:19 +0800 Subject: [PATCH] [HUDI-2520] Fix drop partition issue when sync to hive (#5147) --- .../apache/hudi/hive/ddl/HMSDDLExecutor.java | 7 +++-- .../hudi/hive/ddl/HiveQueryDDLExecutor.java | 8 ++++-- .../hudi/hive/util/HivePartitionUtil.java | 28 +++++++++++++++++-- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java index f2e990535..868f59b4f 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java @@ -238,8 +238,11 @@ public class HMSDDLExecutor implements DDLExecutor { LOG.info("Drop partitions " + partitionsToDrop.size() + " on " + tableName); try { for (String dropPartition : partitionsToDrop) { - String partitionClause = HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, syncConfig); - client.dropPartition(syncConfig.databaseName, tableName, partitionClause, false); + if (HivePartitionUtil.partitionExists(client, tableName, dropPartition, partitionValueExtractor, syncConfig)) { + String partitionClause = + HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, syncConfig); + client.dropPartition(syncConfig.databaseName, tableName, partitionClause, false); + } LOG.info("Drop partition " + dropPartition + " on " + tableName); } } catch (TException e) { diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java index a4debfbab..4b8ceec95 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java @@ -137,8 +137,12 @@ public class HiveQueryDDLExecutor extends QueryBasedDDLExecutor { LOG.info("Drop partitions " + partitionsToDrop.size() + " on " + tableName); try { for (String dropPartition : partitionsToDrop) { - String partitionClause = HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, config); - metaStoreClient.dropPartition(config.databaseName, tableName, partitionClause, false); + if (HivePartitionUtil.partitionExists(metaStoreClient, tableName, dropPartition, partitionValueExtractor, + config)) { + String partitionClause = + HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, config); + metaStoreClient.dropPartition(config.databaseName, tableName, partitionClause, false); + } LOG.info("Drop partition " + dropPartition + " on " + tableName); } } catch (Exception e) { diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HivePartitionUtil.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HivePartitionUtil.java index 27e3a73ce..0258cfc5e 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HivePartitionUtil.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HivePartitionUtil.java @@ -18,15 +18,22 @@ package org.apache.hudi.hive.util; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hudi.common.util.PartitionPathEncodeUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveSyncException; import org.apache.hudi.hive.PartitionValueExtractor; - -import java.util.ArrayList; -import java.util.List; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.thrift.TException; public class HivePartitionUtil { + private static final Logger LOG = LogManager.getLogger(HivePartitionUtil.class); /** * Build String, example as year=2021/month=06/day=25 @@ -48,4 +55,19 @@ public class HivePartitionUtil { } return String.join("/", partBuilder); } + + public static Boolean partitionExists(IMetaStoreClient client, String tableName, String partitionPath, + PartitionValueExtractor partitionValueExtractor, HiveSyncConfig config) { + Partition newPartition; + try { + List partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partitionPath); + newPartition = client.getPartition(config.databaseName, tableName, partitionValues); + } catch (NoSuchObjectException ignored) { + newPartition = null; + } catch (TException e) { + LOG.error("Failed to get partition " + partitionPath, e); + throw new HoodieHiveSyncException("Failed to get partition " + partitionPath, e); + } + return newPartition != null; + } }