lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3030] InProcessLockPovider as default when any async servcies enabled with no lock provider override (#4406)

Browse Source 
* [HUDI-3030] InProcessLockPovider as default when any async servcies enabled with no lock provider override

 - Making InProcessLockProvider as the default lock provider when
   any async services are enabled and when no lock provider is
   explicitly set.

 - This is the workaround for metadata table updates racing with
   async table serice operations

* [HUDI-3030] InProcessLockPovider as default when any async servcies enabled with no lock provider override

 - Renaming isAnyTableServicesInline/Async() to areAnyTableServicesInline/Async()

* [HUDI-3030] InProcessLockPovider as default when any async servcies enabled with no lock provider override

 - Additionally checking for write config properties when verifying
   the lock provider override. Updated the unit test for this case.
This commit is contained in:
Manoj Govindassamy
2022-01-09 19:10:24 -08:00
committed by GitHub
parent 56f93f4ebd
commit 251d4eb3b6
5 changed files with 149 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
View File

@@ -464,7 +464,7 @@ public abstract class AbstractHoodieWriteClient<T extends HoodieRecordPayload, I
}
protected void runTableServicesInline(HoodieTable<T, I, K, O> table, HoodieCommitMetadata metadata, Option<Map<String, String>> extraMetadata) {
if (config.inlineTableServices()) {
if (config.areAnyTableServicesInline()) {
if (config.isMetadataTableEnabled()) {
table.getHoodieView().sync();
}

15
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
View File

@@ -25,6 +25,7 @@ import org.apache.hudi.common.config.LockConfiguration;
import org.apache.hudi.common.config.SerializableConfiguration;
import org.apache.hudi.common.lock.LockProvider;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.config.HoodieLockConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieLockException;
import org.apache.log4j.LogManager;
@@ -42,12 +43,18 @@ public class LockManager implements Serializable, AutoCloseable {
private final HoodieWriteConfig writeConfig;
private final LockConfiguration lockConfiguration;
private final SerializableConfiguration hadoopConf;
private final int maxRetries;
private final long maxWaitTimeInMs;
private volatile LockProvider lockProvider;
public LockManager(HoodieWriteConfig writeConfig, FileSystem fs) {
this.writeConfig = writeConfig;
this.hadoopConf = new SerializableConfiguration(fs.getConf());
this.lockConfiguration = new LockConfiguration(writeConfig.getProps());
maxRetries = lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY,
Integer.parseInt(HoodieLockConfig.LOCK_ACQUIRE_CLIENT_NUM_RETRIES.defaultValue()));
maxWaitTimeInMs = lockConfiguration.getConfig().getLong(LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY,
Long.parseLong(HoodieLockConfig.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS.defaultValue()));
}
public void lock() {
@@ -55,19 +62,17 @@ public class LockManager implements Serializable, AutoCloseable {
LockProvider lockProvider = getLockProvider();
int retryCount = 0;
boolean acquired = false;
int retries = lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY);
long waitTimeInMs = lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY);
while (retryCount <= retries) {
while (retryCount <= maxRetries) {
try {
acquired = lockProvider.tryLock(writeConfig.getLockAcquireWaitTimeoutInMs(), TimeUnit.MILLISECONDS);
if (acquired) {
break;
}
LOG.info("Retrying to acquire lock...");
Thread.sleep(waitTimeInMs);
Thread.sleep(maxWaitTimeInMs);
retryCount++;
} catch (HoodieLockException | InterruptedException e) {
if (retryCount >= retries) {
if (retryCount >= maxRetries) {
throw new HoodieLockException("Unable to acquire lock, lock object ", e);
}
}

34
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -21,12 +21,14 @@ package org.apache.hudi.config;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.bootstrap.BootstrapMode;
import org.apache.hudi.client.transaction.ConflictResolutionStrategy;
import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieCommonConfig;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
@@ -1834,10 +1836,24 @@ public class HoodieWriteConfig extends HoodieConfig {
return WriteConcurrencyMode.fromValue(getString(WRITE_CONCURRENCY_MODE));
}
public Boolean inlineTableServices() {
/**
* Are any table services configured to run inline?
*
* @return True if any table services are configured to run inline, false otherwise.
*/
public Boolean areAnyTableServicesInline() {
return inlineClusteringEnabled() || inlineCompactionEnabled() || isAutoClean();
}
/**
* Are any table services configured to run async?
*
* @return True if any table services are configured to run async, false otherwise.
*/
public Boolean areAnyTableServicesAsync() {
return isAsyncClusteringEnabled() || !inlineCompactionEnabled() || isAsyncClean();
}
public String getPreCommitValidators() {
return getString(HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES);
}
@@ -2267,13 +2283,25 @@ public class HoodieWriteConfig extends HoodieConfig {
HoodiePayloadConfig.newBuilder().fromProperties(writeConfig.getProps()).build());
writeConfig.setDefaultOnCondition(!isMetadataConfigSet,
HoodieMetadataConfig.newBuilder().withEngineType(engineType).fromProperties(writeConfig.getProps()).build());
writeConfig.setDefaultOnCondition(!isLockConfigSet,
HoodieLockConfig.newBuilder().fromProperties(writeConfig.getProps()).build());
writeConfig.setDefaultOnCondition(!isPreCommitValidationConfigSet,
HoodiePreCommitValidatorConfig.newBuilder().fromProperties(writeConfig.getProps()).build());
writeConfig.setDefaultOnCondition(!isLayoutConfigSet,
HoodieLayoutConfig.newBuilder().fromProperties(writeConfig.getProps()).build());
writeConfig.setDefaultValue(TIMELINE_LAYOUT_VERSION_NUM, String.valueOf(TimelineLayoutVersion.CURR_VERSION));
// Async table services can update the metadata table and a lock provider is
// needed to guard against any concurrent table write operations. If user has
// not configured any lock provider, let's use the InProcess lock provider.
final TypedProperties writeConfigProperties = writeConfig.getProps();
final boolean isLockProviderPropertySet = writeConfigProperties.containsKey(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME)
|| writeConfigProperties.containsKey(HoodieLockConfig.LOCK_PROVIDER_CLASS_PROP);
if (!isLockConfigSet) {
HoodieLockConfig.Builder lockConfigBuilder = HoodieLockConfig.newBuilder().fromProperties(writeConfig.getProps());
if (!isLockProviderPropertySet && writeConfig.areAnyTableServicesAsync()) {
lockConfigBuilder.withLockProvider(InProcessLockProvider.class);
}
writeConfig.setDefault(lockConfigBuilder.build());
}
}
private void validate() {