lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Fix BindException when running tests of shared machines. (#2070)

Browse Source 
When unit tests are run on shared machines (e.g. jenkins cluster), the unit tests sometimes fail due to BindException in starting HDFS Cluster. This is because the port chosen may have been bound by another process using the same machine. The fix is to retry the port selection a few times.
This commit is contained in:
Prashant Wason
2020-09-07 19:30:45 -07:00
committed by GitHub
parent 83e39e2b17
commit fe7c9e71eb
1 changed files with 26 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java
View File

@@ -31,6 +31,7 @@ import org.apache.log4j.Logger;
import java.io.File;
import java.io.IOException;
import java.net.BindException;
import java.nio.file.Files;
import java.util.Objects;
@@ -72,20 +73,32 @@ public class HdfsTestService {
FileIOUtils.deleteDirectory(file);
}
int namenodeRpcPort = NetworkTestUtils.nextFreePort();
int datanodePort = NetworkTestUtils.nextFreePort();
int datanodeIpcPort = NetworkTestUtils.nextFreePort();
int datanodeHttpPort = NetworkTestUtils.nextFreePort();
int loop = 0;
while (true) {
try {
int namenodeRpcPort = NetworkTestUtils.nextFreePort();
int datanodePort = NetworkTestUtils.nextFreePort();
int datanodeIpcPort = NetworkTestUtils.nextFreePort();
int datanodeHttpPort = NetworkTestUtils.nextFreePort();
// Configure and start the HDFS cluster
// boolean format = shouldFormatDFSCluster(localDFSLocation, clean);
String bindIP = "127.0.0.1";
configureDFSCluster(hadoopConf, localDFSLocation, bindIP, namenodeRpcPort,
datanodePort, datanodeIpcPort, datanodeHttpPort);
miniDfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(1).format(format).checkDataNodeAddrConfig(true)
.checkDataNodeHostConfig(true).build();
LOG.info("HDFS Minicluster service started.");
return miniDfsCluster;
// Configure and start the HDFS cluster
// boolean format = shouldFormatDFSCluster(localDFSLocation, clean);
String bindIP = "127.0.0.1";
configureDFSCluster(hadoopConf, localDFSLocation, bindIP, namenodeRpcPort,
datanodePort, datanodeIpcPort, datanodeHttpPort);
miniDfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(1).format(format).checkDataNodeAddrConfig(true)
.checkDataNodeHostConfig(true).build();
LOG.info("HDFS Minicluster service started.");
return miniDfsCluster;
} catch (BindException ex) {
++loop;
if (loop < 5) {
stop();
} else {
throw ex;
}
}
}
}
public void stop() {