lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3780] improve drop partitions (#5178)

Browse Source
This commit is contained in:
ForwardXu
2022-04-05 11:52:33 +08:00
committed by GitHub
parent b28f0d6ceb
commit 3449e86989
14 changed files with 216 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/AbstractSyncHoodieClient.java
View File

@@ -18,6 +18,15 @@
package org.apache.hudi.sync.common;
import java.io.Serializable;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -29,21 +38,11 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.TimelineUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.metadata.HoodieTableMetadataUtil;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.parquet.schema.MessageType;
import java.io.Serializable;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public abstract class AbstractSyncHoodieClient implements AutoCloseable {
private static final Logger LOG = LogManager.getLogger(AbstractSyncHoodieClient.class);
@@ -113,7 +112,7 @@ public abstract class AbstractSyncHoodieClient implements AutoCloseable {
public abstract void updatePartitionsToTable(String tableName, List<String> changedPartitions);
public abstract void dropPartitionsToTable(String tableName, List<String> partitionsToDrop);
public abstract void dropPartitions(String tableName, List<String> partitionsToDrop);
public void updateTableProperties(String tableName, Map<String, String> tableProperties) {}
@@ -170,8 +169,7 @@ public abstract class AbstractSyncHoodieClient implements AutoCloseable {
public boolean isDropPartition() {
try {
Option<HoodieCommitMetadata> hoodieCommitMetadata;
hoodieCommitMetadata = new TableSchemaResolver(metaClient).getLatestCommitMetadata();
Option<HoodieCommitMetadata> hoodieCommitMetadata = HoodieTableMetadataUtil.getLatestCommitMetadata(metaClient);
if (hoodieCommitMetadata.isPresent()
&& WriteOperationType.DELETE_PARTITION.equals(hoodieCommitMetadata.get().getOperationType())) {