lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3064][HUDI-3054] FileSystemBasedLockProviderTestClass tryLock fix and TestHoodieClientMultiWriter test fixes (#4384)

Browse Source 
- Made FileSystemBasedLockProviderTestClass thread safe and fixed the
   tryLock retry logic.

 - Made TestHoodieClientMultiWriter. testHoodieClientBasicMultiWriter
   deterministic in verifying the HoodieWriteConflictException.
This commit is contained in:
Manoj Govindassamy
2021-12-19 10:31:02 -08:00
committed by GitHub
parent 03f71ef1a2
commit 4a48f99a59
2 changed files with 121 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/FileSystemBasedLockProviderTestClass.java
View File

@@ -36,38 +36,37 @@ import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_NUM_R
import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY;
/**
* This lock provider is used for testing purposes only. It provides a simple file system based lock using HDFS atomic
* create operation. This lock does not support cleaning/expiring the lock after a failed write hence cannot be used
* in production environments.
* This lock provider is used for testing purposes only. It provides a simple file system based lock
* using filesystem's atomic create operation. This lock does not support cleaning/expiring the lock
* after a failed write. Must not be used in production environments.
*/
public class FileSystemBasedLockProviderTestClass implements LockProvider<String>, Serializable {
private static final String LOCK_NAME = "acquired";
private static final String LOCK = "lock";
private String lockPath;
private final int retryMaxCount;
private final int retryWaitTimeMs;
private transient FileSystem fs;
private transient Path lockFile;
protected LockConfiguration lockConfiguration;
public FileSystemBasedLockProviderTestClass(final LockConfiguration lockConfiguration, final Configuration configuration) {
this.lockConfiguration = lockConfiguration;
this.lockPath = lockConfiguration.getConfig().getString(FILESYSTEM_LOCK_PATH_PROP_KEY);
this.fs = FSUtils.getFs(this.lockPath, configuration);
}
public void acquireLock() {
try {
fs.create(new Path(lockPath + "/" + LOCK_NAME), false).close();
} catch (IOException e) {
throw new HoodieIOException("Failed to acquire lock", e);
}
final String lockDirectory = lockConfiguration.getConfig().getString(FILESYSTEM_LOCK_PATH_PROP_KEY);
this.retryWaitTimeMs = lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY);
this.retryMaxCount = lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY);
this.lockFile = new Path(lockDirectory + "/" + LOCK);
this.fs = FSUtils.getFs(this.lockFile.toString(), configuration);
}
@Override
public void close() {
try {
fs.delete(new Path(lockPath + "/" + LOCK_NAME), true);
} catch (IOException e) {
throw new HoodieLockException("Unable to release lock", e);
synchronized (LOCK) {
try {
fs.delete(this.lockFile, true);
} catch (IOException e) {
throw new HoodieLockException("Unable to release lock: " + getLock(), e);
}
}
}
@@ -75,39 +74,45 @@ public class FileSystemBasedLockProviderTestClass implements LockProvider<String
public boolean tryLock(long time, TimeUnit unit) {
try {
int numRetries = 0;
while (fs.exists(new Path(lockPath + "/" + LOCK_NAME))
&& (numRetries++ <= lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY))) {
Thread.sleep(lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY));
}
synchronized (LOCK_NAME) {
if (fs.exists(new Path(lockPath + "/" + LOCK_NAME))) {
return false;
synchronized (LOCK) {
while (fs.exists(this.lockFile)) {
LOCK.wait(retryWaitTimeMs);
numRetries++;
if (numRetries > retryMaxCount) {
return false;
}
}
acquireLock();
return fs.exists(this.lockFile);
}
return true;
} catch (IOException | InterruptedException e) {
throw new HoodieLockException("Failed to acquire lock", e);
throw new HoodieLockException("Failed to acquire lock: " + getLock(), e);
}
}
@Override
public void unlock() {
try {
if (fs.exists(new Path(lockPath + "/" + LOCK_NAME))) {
fs.delete(new Path(lockPath + "/" + LOCK_NAME), true);
synchronized (LOCK) {
try {
if (fs.exists(this.lockFile)) {
fs.delete(this.lockFile, true);
}
} catch (IOException io) {
throw new HoodieIOException("Unable to delete lock " + getLock() + "on disk", io);
}
} catch (IOException io) {
throw new HoodieIOException("Unable to delete lock on disk", io);
}
}
@Override
public String getLock() {
return this.lockFile.toString();
}
private void acquireLock() {
try {
return fs.listStatus(new Path(lockPath))[0].getPath().toString();
} catch (Exception e) {
throw new HoodieLockException("Failed to retrieve lock status from lock path " + lockPath);
fs.create(this.lockFile, false).close();
} catch (IOException e) {
throw new HoodieIOException("Failed to acquire lock: " + getLock(), e);
}
}
}