lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3252] Avoid creating empty requestedReplaceCommit in the startCommit method (#4515)

Browse Source
This commit is contained in:
董可伦
2022-01-18 06:28:18 +08:00
committed by GitHub
parent d36533735f
commit 20e7983866
4 changed files with 22 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
View File

@@ -828,8 +828,13 @@ public abstract class AbstractHoodieWriteClient<T extends HoodieRecordPayload, I
if (config.getFailedWritesCleanPolicy().isLazy()) {
this.heartbeatClient.start(instantTime);
}
metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(HoodieInstant.State.REQUESTED, actionType,
instantTime));
if (actionType.equals(HoodieTimeline.REPLACE_COMMIT_ACTION)) {
metaClient.getActiveTimeline().createRequestedReplaceCommit(instantTime, actionType);
} else {
metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(HoodieInstant.State.REQUESTED, actionType,
instantTime));
}
}
/**

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataConversionUtils.java
View File

@@ -134,7 +134,7 @@ public class MetadataConversionUtils {
return Option.of(HoodieCommitMetadata.fromBytes(inflightContent.get(), HoodieCommitMetadata.class));
}
public static Option<HoodieRequestedReplaceMetadata> getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant instant) throws IOException {
private static Option<HoodieRequestedReplaceMetadata> getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant instant) throws IOException {
Option<byte[]> requestedContent = metaClient.getActiveTimeline().getInstantDetails(instant);
if (!requestedContent.isPresent() || requestedContent.get().length == 0) {
// requested commit files can be empty in some certain cases, e.g. insert_overwrite or insert_overwrite_table.

13
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
View File

@@ -18,6 +18,7 @@
package org.apache.hudi.common.table.timeline;
import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.FileIOUtils;
@@ -156,6 +157,18 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
createFileInMetaPath(instant.getFileName(), Option.empty(), false);
}
public void createRequestedReplaceCommit(String instantTime, String actionType) {
try {
HoodieInstant instant = new HoodieInstant(State.REQUESTED, actionType, instantTime);
LOG.info("Creating a new instant " + instant);
// Create the request replace file
createFileInMetaPath(instant.getFileName(),
TimelineMetadataUtils.serializeRequestedReplaceMetadata(new HoodieRequestedReplaceMetadata()), false);
} catch (IOException e) {
throw new HoodieIOException("Error create requested replace commit ", e);
}
}
public void saveAsComplete(HoodieInstant instant, Option<byte[]> data) {
LOG.info("Marking instant complete " + instant);
ValidationUtils.checkArgument(instant.isInflight(),

2
hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
View File

@@ -77,7 +77,7 @@ public class ClusteringUtils {
* @return
* @throws IOException
*/
public static Option<HoodieRequestedReplaceMetadata> getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant pendingReplaceInstant) throws IOException {
private static Option<HoodieRequestedReplaceMetadata> getRequestedReplaceMetadata(HoodieTableMetaClient metaClient, HoodieInstant pendingReplaceInstant) throws IOException {
final HoodieInstant requestedInstant;
if (!pendingReplaceInstant.isRequested()) {
// inflight replacecommit files don't have clustering plan.