lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2579] Make deltastreamer checkpoint state merging more explicit (#3820)

Browse Source 
Co-authored-by: Sivabalan Narayanan <n.siva.b@gmail.com>
This commit is contained in:
davehagman
2021-11-09 17:37:59 -05:00
committed by GitHub
parent 2f95967dfe
commit dfe3b84715
6 changed files with 195 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.client.utils;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.hudi.client.transaction.ConflictResolutionStrategy;
@@ -53,8 +53,12 @@ public class TransactionUtils {
* @return
* @throws HoodieWriteConflictException
*/
public static Option<HoodieCommitMetadata> resolveWriteConflictIfAny(final HoodieTable table, final Option<HoodieInstant> currentTxnOwnerInstant,
final Option<HoodieCommitMetadata> thisCommitMetadata, final HoodieWriteConfig config, Option<HoodieInstant> lastCompletedTxnOwnerInstant) throws HoodieWriteConflictException {
public static Option<HoodieCommitMetadata> resolveWriteConflictIfAny(
final HoodieTable table,
final Option<HoodieInstant> currentTxnOwnerInstant,
final Option<HoodieCommitMetadata> thisCommitMetadata,
final HoodieWriteConfig config,
Option<HoodieInstant> lastCompletedTxnOwnerInstant) throws HoodieWriteConflictException {
if (config.getWriteConcurrencyMode().supportsOptimisticConcurrencyControl()) {
ConflictResolutionStrategy resolutionStrategy = config.getWriteConflictResolutionStrategy();
Stream<HoodieInstant> instantStream = resolutionStrategy.getCandidateInstants(table.getActiveTimeline(), currentTxnOwnerInstant.get(), lastCompletedTxnOwnerInstant);
@@ -72,8 +76,11 @@ public class TransactionUtils {
}
});
LOG.info("Successfully resolved conflicts, if any");
// carry over necessary metadata from latest commit metadata
overrideWithLatestCommitMetadata(table.getMetaClient(), thisOperation.getCommitMetadataOption(), currentTxnOwnerInstant, Arrays.asList(config.getWriteMetaKeyPrefixes().split(",")));
if (config.mergeDeltastreamerStateFromPreviousCommit()) {
mergeCheckpointStateFromPreviousCommit(table.getMetaClient(), thisOperation.getCommitMetadataOption());
}
return thisOperation.getCommitMetadataOption();
}
return thisCommitMetadata;
@@ -111,16 +118,27 @@ public class TransactionUtils {
}
}
// override the current metadata with the metadata from the latest instant for the specified key prefixes
private static void overrideWithLatestCommitMetadata(HoodieTableMetaClient metaClient, Option<HoodieCommitMetadata> thisMetadata,
Option<HoodieInstant> thisInstant, List<String> keyPrefixes) {
protected static void mergeCheckpointStateFromPreviousCommit(HoodieTableMetaClient metaClient, Option<HoodieCommitMetadata> thisMetadata) {
overrideWithLatestCommitMetadata(metaClient, thisMetadata, Collections.singletonList(HoodieWriteConfig.DELTASTREAMER_CHECKPOINT_KEY));
}
/**
* Generic method allowing us to override the current metadata with the metadata from
* the latest instant for the specified key prefixes.
* @param metaClient
* @param thisMetadata
* @param keyPrefixes The key prefixes to merge from the previous commit
*/
private static void overrideWithLatestCommitMetadata(HoodieTableMetaClient metaClient,
Option<HoodieCommitMetadata> thisMetadata,
List<String> keyPrefixes) {
if (keyPrefixes.size() == 1 && keyPrefixes.get(0).length() < 1) {
return;
}
Option<Pair<HoodieInstant, Map<String, String>>> lastInstant = getLastCompletedTxnInstantAndMetadata(metaClient);
if (lastInstant.isPresent() && thisMetadata.isPresent()) {
Stream<String> keys = thisMetadata.get().getExtraMetadata().keySet().stream();
keyPrefixes.stream().forEach(keyPrefix -> keys
Stream<String> lastCommitMetadataKeys = lastInstant.get().getRight().keySet().stream();
keyPrefixes.stream().forEach(keyPrefix -> lastCommitMetadataKeys
.filter(key -> key.startsWith(keyPrefix))
.forEach(key -> thisMetadata.get().getExtraMetadata().put(key, lastInstant.get().getRight().get(key))));
}

39
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -85,6 +85,10 @@ public class HoodieWriteConfig extends HoodieConfig {
private static final long serialVersionUID = 0L;
// This is a constant as is should never be changed via config (will invalidate previous commits)
// It is here so that both the client and deltastreamer use the same reference
public static final String DELTASTREAMER_CHECKPOINT_KEY = "deltastreamer.checkpoint.key";
public static final ConfigProperty<String> TBL_NAME = ConfigProperty
.key("hoodie.table.name")
.noDefaultValue()
@@ -368,11 +372,13 @@ public class HoodieWriteConfig extends HoodieConfig {
+ "OPTIMISTIC_CONCURRENCY_CONTROL: Multiple writers can operate on the table and exactly one of them succeed "
+ "if a conflict (writes affect the same file group) is detected.");
public static final ConfigProperty<String> WRITE_META_KEY_PREFIXES = ConfigProperty
.key("hoodie.write.meta.key.prefixes")
.defaultValue("")
.withDocumentation("Comma separated metadata key prefixes to override from latest commit "
+ "during overlapping commits via multi writing");
public static final ConfigProperty<Boolean> WRITE_CONCURRENCY_MERGE_DELTASTREAMER_STATE = ConfigProperty
.key("hoodie.write.concurrency.merge.deltastreamer.state")
.defaultValue(false)
.withAlternatives("hoodie.write.meta.key.prefixes")
.withDocumentation("If enabled, this writer will merge Deltastreamer state from the previous checkpoint in order to allow both realtime "
+ "and batch writers to ingest into a single table. This should not be enabled on Deltastreamer writers. Enabling this config means,"
+ "for a spark writer, deltastreamer checkpoint will be copied over from previous commit to the current one.");
/**
* Currently the use this to specify the write schema.
@@ -783,16 +789,6 @@ public class HoodieWriteConfig extends HoodieConfig {
*/
@Deprecated
public static final String DEFAULT_WRITE_CONCURRENCY_MODE = WRITE_CONCURRENCY_MODE.defaultValue();
/**
* @deprecated Use {@link #WRITE_META_KEY_PREFIXES} and its methods instead
*/
@Deprecated
public static final String WRITE_META_KEY_PREFIXES_PROP = WRITE_META_KEY_PREFIXES.key();
/**
* @deprecated Use {@link #WRITE_META_KEY_PREFIXES} and its methods instead
*/
@Deprecated
public static final String DEFAULT_WRITE_META_KEY_PREFIXES = WRITE_META_KEY_PREFIXES.defaultValue();
/**
* @deprecated Use {@link #ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE} and its methods instead
*/
@@ -1764,12 +1760,12 @@ public class HoodieWriteConfig extends HoodieConfig {
return WriteConcurrencyMode.fromValue(getString(WRITE_CONCURRENCY_MODE));
}
public Boolean inlineTableServices() {
return inlineClusteringEnabled() || inlineCompactionEnabled() || isAutoClean();
public Boolean mergeDeltastreamerStateFromPreviousCommit() {
return getBoolean(HoodieWriteConfig.WRITE_CONCURRENCY_MERGE_DELTASTREAMER_STATE);
}
public String getWriteMetaKeyPrefixes() {
return getString(WRITE_META_KEY_PREFIXES);
public Boolean inlineTableServices() {
return inlineClusteringEnabled() || inlineCompactionEnabled() || isAutoClean();
}
public String getPreCommitValidators() {
@@ -2131,11 +2127,6 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
public Builder withWriteMetaKeyPrefixes(String writeMetaKeyPrefixes) {
writeConfig.setValue(WRITE_META_KEY_PREFIXES, writeMetaKeyPrefixes);
return this;
}
public Builder withPopulateMetaFields(boolean populateMetaFields) {
writeConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields));
return this;