diff --git a/docker/demo/compaction-bootstrap.commands b/docker/demo/compaction-bootstrap.commands new file mode 100644 index 000000000..6c246be74 --- /dev/null +++ b/docker/demo/compaction-bootstrap.commands @@ -0,0 +1,21 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +connect --path /user/hive/warehouse/stock_ticks_mor_bs +compactions show all +compaction schedule --hoodieConfigs hoodie.compact.inline.max.delta.commits=1 +compaction run --parallelism 2 --sparkMemory 1G --schemaFilePath /var/demo/config/schema.avsc --retry 1 diff --git a/docker/demo/compaction.commands b/docker/demo/compaction.commands index 6abdad743..a8baaff3e 100644 --- a/docker/demo/compaction.commands +++ b/docker/demo/compaction.commands @@ -19,7 +19,3 @@ connect --path /user/hive/warehouse/stock_ticks_mor compactions show all compaction schedule --hoodieConfigs hoodie.compact.inline.max.delta.commits=1 compaction run --parallelism 2 --sparkMemory 1G --schemaFilePath /var/demo/config/schema.avsc --retry 1 -connect --path /user/hive/warehouse/stock_ticks_mor_bs -compactions show all -compaction schedule --hoodieConfigs hoodie.compact.inline.max.delta.commits=1 -compaction run --parallelism 2 --sparkMemory 1G --schemaFilePath /var/demo/config/schema.avsc --retry 1 diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java index 04231036c..d0b32eec3 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java @@ -115,11 +115,9 @@ public abstract class ITTestBase { } static String getPrestoConsoleCommand(String commandFile) { - StringBuilder builder = new StringBuilder().append("presto --server " + PRESTO_COORDINATOR_URL) + return new StringBuilder().append("presto --server " + PRESTO_COORDINATOR_URL) .append(" --catalog hive --schema default") - .append(" -f " + commandFile); - System.out.println("Presto comamnd " + builder.toString()); - return builder.toString(); + .append(" -f " + commandFile).toString(); } @BeforeEach @@ -166,14 +164,14 @@ public abstract class ITTestBase { boolean completed = dockerClient.execStartCmd(createCmdResponse.getId()).withDetach(false).withTty(false).exec(callback) - .awaitCompletion(900, SECONDS); + .awaitCompletion(540, SECONDS); if (!completed) { callback.getStderr().flush(); callback.getStdout().flush(); LOG.error("\n\n ###### Timed Out Command : " + Arrays.asList(command)); LOG.error("\n\n ###### Stderr of timed-out command #######\n" + callback.getStderr().toString()); - LOG.error("\n\n ###### stdout of timed-out command #######\n" + callback.getStderr().toString()); - throw new TimeoutException("Command " + command + " has been running for more than 15 minutes. " + LOG.error("\n\n ###### stdout of timed-out command #######\n" + callback.getStdout().toString()); + throw new TimeoutException("Command " + command + " has been running for more than 9 minutes. " + "Killing and failing !!"); } int exitCode = dockerClient.inspectExecCmd(createCmdResponse.getId()).exec().getExitCode(); diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java index d2a084172..eb608df87 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java @@ -65,6 +65,7 @@ public class ITTestHoodieDemo extends ITTestBase { private static final String MIN_COMMIT_TIME_MOR_SCRIPT = HOODIE_WS_ROOT + "/docker/demo/get_min_commit_time_mor.sh"; private static final String HUDI_CLI_TOOL = HOODIE_WS_ROOT + "/hudi-cli/hudi-cli.sh"; private static final String COMPACTION_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/compaction.commands"; + private static final String COMPACTION_BOOTSTRAP_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/compaction-bootstrap.commands"; private static final String SPARKSQL_BS_PREP_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/sparksql-bootstrap-prep-source.commands"; private static final String SPARKSQL_BATCH1_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/sparksql-batch1.commands"; private static final String SPARKSQL_BATCH2_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/sparksql-batch2.commands"; @@ -407,5 +408,6 @@ public class ITTestHoodieDemo extends ITTestBase { private void scheduleAndRunCompaction() throws Exception { executeCommandStringInDocker(ADHOC_1_CONTAINER, HUDI_CLI_TOOL + " --cmdfile " + COMPACTION_COMMANDS, true); + executeCommandStringInDocker(ADHOC_1_CONTAINER, HUDI_CLI_TOOL + " --cmdfile " + COMPACTION_BOOTSTRAP_COMMANDS, true); } } diff --git a/pom.xml b/pom.xml index 1d25e6802..cce96d2ce 100644 --- a/pom.xml +++ b/pom.xml @@ -1121,7 +1121,6 @@ ${skipITs} - **/testsuite/**/Test*.java **/IT*.java