lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4065] Add FileBasedLockProvider (#6071)

Browse Source
This commit is contained in:
冯健
2022-07-19 07:52:47 +08:00
committed by GitHub
parent 1959b843b7
commit 382d19e85b
5 changed files with 349 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java Normal file
View File

@@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hudi.client.transaction.lock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.config.LockConfiguration;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.lock.LockProvider;
import org.apache.hudi.common.lock.LockState;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieLockException;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.io.Serializable;
import java.util.concurrent.TimeUnit;
import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_EXPIRE_PROP_KEY;
import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PATH_PROP_KEY;
/**
* A FileSystem based lock. This {@link LockProvider} implementation allows to lock table operations
* using DFS. Users might need to manually clean the Locker's path if writeClient crash and never run again.
* NOTE: This only works for DFS with atomic create/delete operation
*/
public class FileSystemBasedLockProvider implements LockProvider<String>, Serializable {
private static final Logger LOG = LogManager.getLogger(FileSystemBasedLockProvider.class);
private static final String LOCK_FILE_NAME = "lock";
private final int lockTimeoutMinutes;
private transient FileSystem fs;
private transient Path lockFile;
protected LockConfiguration lockConfiguration;
public FileSystemBasedLockProvider(final LockConfiguration lockConfiguration, final Configuration configuration) {
checkRequiredProps(lockConfiguration);
this.lockConfiguration = lockConfiguration;
String lockDirectory = lockConfiguration.getConfig().getString(FILESYSTEM_LOCK_PATH_PROP_KEY, null);
if (StringUtils.isNullOrEmpty(lockDirectory)) {
lockDirectory = lockConfiguration.getConfig().getString(HoodieWriteConfig.BASE_PATH.key())
+ Path.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME;
}
this.lockTimeoutMinutes = lockConfiguration.getConfig().getInteger(FILESYSTEM_LOCK_EXPIRE_PROP_KEY);
this.lockFile = new Path(lockDirectory + Path.SEPARATOR + LOCK_FILE_NAME);
this.fs = FSUtils.getFs(this.lockFile.toString(), configuration);
}
@Override
public void close() {
synchronized (LOCK_FILE_NAME) {
try {
fs.delete(this.lockFile, true);
} catch (IOException e) {
throw new HoodieLockException(generateLogStatement(LockState.FAILED_TO_RELEASE), e);
}
}
}
@Override
public boolean tryLock(long time, TimeUnit unit) {
try {
synchronized (LOCK_FILE_NAME) {
// Check whether lock is already expired, if so try to delete lock file
if (fs.exists(this.lockFile) && checkIfExpired()) {
fs.delete(this.lockFile, true);
}
acquireLock();
return fs.exists(this.lockFile);
}
} catch (IOException | HoodieIOException e) {
LOG.info(generateLogStatement(LockState.FAILED_TO_ACQUIRE), e);
return false;
}
}
@Override
public void unlock() {
synchronized (LOCK_FILE_NAME) {
try {
if (fs.exists(this.lockFile)) {
fs.delete(this.lockFile, true);
}
} catch (IOException io) {
throw new HoodieIOException(generateLogStatement(LockState.FAILED_TO_RELEASE), io);
}
}
}
@Override
public String getLock() {
return this.lockFile.toString();
}
private boolean checkIfExpired() {
if (lockTimeoutMinutes == 0) {
return false;
}
try {
long modificationTime = fs.getFileStatus(this.lockFile).getModificationTime();
if (System.currentTimeMillis() - modificationTime > lockTimeoutMinutes * 60 * 1000) {
return true;
}
} catch (IOException | HoodieIOException e) {
LOG.error(generateLogStatement(LockState.ALREADY_RELEASED) + " failed to get lockFile's modification time", e);
}
return false;
}
private void acquireLock() {
try {
fs.create(this.lockFile, false).close();
} catch (IOException e) {
throw new HoodieIOException(generateLogStatement(LockState.FAILED_TO_ACQUIRE), e);
}
}
protected String generateLogStatement(LockState state) {
return StringUtils.join(state.name(), " lock at: ", getLock());
}
private void checkRequiredProps(final LockConfiguration config) {
ValidationUtils.checkArgument(config.getConfig().getString(FILESYSTEM_LOCK_PATH_PROP_KEY, null) != null
|| config.getConfig().getString(HoodieWriteConfig.BASE_PATH.key(), null) != null);
ValidationUtils.checkArgument(config.getConfig().getInteger(FILESYSTEM_LOCK_EXPIRE_PROP_KEY) >= 0);
}
}

9
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
View File

@@ -36,6 +36,7 @@ import static org.apache.hudi.common.config.LockConfiguration.DEFAULT_LOCK_ACQUI
import static org.apache.hudi.common.config.LockConfiguration.DEFAULT_LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS;
import static org.apache.hudi.common.config.LockConfiguration.DEFAULT_ZK_CONNECTION_TIMEOUT_MS;
import static org.apache.hudi.common.config.LockConfiguration.DEFAULT_ZK_SESSION_TIMEOUT_MS;
import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_EXPIRE_PROP_KEY;
import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PATH_PROP_KEY;
import static org.apache.hudi.common.config.LockConfiguration.HIVE_DATABASE_NAME_PROP_KEY;
import static org.apache.hudi.common.config.LockConfiguration.HIVE_METASTORE_URI_PROP_KEY;
@@ -106,7 +107,13 @@ public class HoodieLockConfig extends HoodieConfig {
.key(FILESYSTEM_LOCK_PATH_PROP_KEY)
.noDefaultValue()
.sinceVersion("0.8.0")
.withDocumentation("For DFS based lock providers, path to store the locks under.");
.withDocumentation("For DFS based lock providers, path to store the locks under. use Table's meta path as default");
public static final ConfigProperty<Integer> FILESYSTEM_LOCK_EXPIRE = ConfigProperty
.key(FILESYSTEM_LOCK_EXPIRE_PROP_KEY)
.defaultValue(0)
.sinceVersion("0.12.0")
.withDocumentation("For DFS based lock providers, expire time in minutes, must be a nonnegative number, default means no expire");
public static final ConfigProperty<String> HIVE_DATABASE_NAME = ConfigProperty
.key(HIVE_DATABASE_NAME_PROP_KEY)