lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2520] Fix drop partition issue when sync to hive (#5147)

Browse Source
This commit is contained in:
ForwardXu
2022-03-30 02:28:19 +08:00
committed by GitHub
parent fcb003ec76
commit 0802510ca9
3 changed files with 36 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java
View File

@@ -238,8 +238,11 @@ public class HMSDDLExecutor implements DDLExecutor {
LOG.info("Drop partitions " + partitionsToDrop.size() + " on " + tableName); LOG.info("Drop partitions " + partitionsToDrop.size() + " on " + tableName);
try { try {
for (String dropPartition : partitionsToDrop) { for (String dropPartition : partitionsToDrop) {
String partitionClause = HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, syncConfig); if (HivePartitionUtil.partitionExists(client, tableName, dropPartition, partitionValueExtractor, syncConfig)) {
client.dropPartition(syncConfig.databaseName, tableName, partitionClause, false); String partitionClause =
HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, syncConfig);
client.dropPartition(syncConfig.databaseName, tableName, partitionClause, false);
}
LOG.info("Drop partition " + dropPartition + " on " + tableName); LOG.info("Drop partition " + dropPartition + " on " + tableName);
} }
} catch (TException e) { } catch (TException e) {

8
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java
View File

@@ -137,8 +137,12 @@ public class HiveQueryDDLExecutor extends QueryBasedDDLExecutor {
LOG.info("Drop partitions " + partitionsToDrop.size() + " on " + tableName); LOG.info("Drop partitions " + partitionsToDrop.size() + " on " + tableName);
try { try {
for (String dropPartition : partitionsToDrop) { for (String dropPartition : partitionsToDrop) {
String partitionClause = HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, config); if (HivePartitionUtil.partitionExists(metaStoreClient, tableName, dropPartition, partitionValueExtractor,
metaStoreClient.dropPartition(config.databaseName, tableName, partitionClause, false); config)) {
String partitionClause =
HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, config);
metaStoreClient.dropPartition(config.databaseName, tableName, partitionClause, false);
}
LOG.info("Drop partition " + dropPartition + " on " + tableName); LOG.info("Drop partition " + dropPartition + " on " + tableName);
} }
} catch (Exception e) { } catch (Exception e) {

28
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HivePartitionUtil.java
View File

@@ -18,15 +18,22 @@
package org.apache.hudi.hive.util; package org.apache.hudi.hive.util;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hudi.common.util.PartitionPathEncodeUtils; import org.apache.hudi.common.util.PartitionPathEncodeUtils;
import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.HoodieHiveSyncException;
import org.apache.hudi.hive.PartitionValueExtractor; import org.apache.hudi.hive.PartitionValueExtractor;
import org.apache.log4j.LogManager;
import java.util.ArrayList; import org.apache.log4j.Logger;
import java.util.List; import org.apache.thrift.TException;
public class HivePartitionUtil { public class HivePartitionUtil {
private static final Logger LOG = LogManager.getLogger(HivePartitionUtil.class);
/** /**
* Build String, example as year=2021/month=06/day=25 * Build String, example as year=2021/month=06/day=25
@@ -48,4 +55,19 @@ public class HivePartitionUtil {
} }
return String.join("/", partBuilder); return String.join("/", partBuilder);
} }
public static Boolean partitionExists(IMetaStoreClient client, String tableName, String partitionPath,
PartitionValueExtractor partitionValueExtractor, HiveSyncConfig config) {
Partition newPartition;
try {
List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partitionPath);
newPartition = client.getPartition(config.databaseName, tableName, partitionValues);
} catch (NoSuchObjectException ignored) {
newPartition = null;
} catch (TException e) {
LOG.error("Failed to get partition " + partitionPath, e);
throw new HoodieHiveSyncException("Failed to get partition " + partitionPath, e);
}
return newPartition != null;
}
} }