From 61e027fadda4cefc56a18bc284267d34b336f6cc Mon Sep 17 00:00:00 2001 From: Balaji Varadarajan Date: Wed, 5 Aug 2020 07:46:34 -0700 Subject: [PATCH] [MINOR] Adding timeout for each command execution in docker and capture output. This will help get stdout/stderr of stuck commands (#1918) --- .../org/apache/hudi/integ/ITTestBase.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java index 7b70857a9..a5e6ed965 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java @@ -18,6 +18,8 @@ package org.apache.hudi.integ; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.collection.Pair; @@ -158,8 +160,21 @@ public abstract class ITTestBase { ExecCreateCmdResponse createCmdResponse = cmd.exec(); TestExecStartResultCallback callback = new TestExecStartResultCallback(new ByteArrayOutputStream(), new ByteArrayOutputStream()); - dockerClient.execStartCmd(createCmdResponse.getId()).withDetach(false).withTty(false).exec(callback) - .awaitCompletion(); + // Each execution of command(s) in docker should not be more than 15 mins. Otherwise, it is deemed stuck. We will + // try to capture stdout and stderr of the stuck process. + + boolean completed = + dockerClient.execStartCmd(createCmdResponse.getId()).withDetach(false).withTty(false).exec(callback) + .awaitCompletion(900, SECONDS); + if (!completed) { + callback.getStderr().flush(); + callback.getStdout().flush(); + LOG.error("\n\n ###### Timed Out Command : " + Arrays.asList(command)); + LOG.error("\n\n ###### Stderr of timed-out command #######\n" + callback.getStderr().toString()); + LOG.error("\n\n ###### stdout of timed-out command #######\n" + callback.getStderr().toString()); + throw new TimeoutException("Command " + command + " has been running for more than 15 minutes. " + + "Killing and failing !!"); + } int exitCode = dockerClient.inspectExecCmd(createCmdResponse.getId()).exec().getExitCode(); LOG.info("Exit code for command : " + exitCode); if (exitCode != 0) {