diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java index dc5b3797f..a4059e16b 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java @@ -115,4 +115,16 @@ public class HoodieCLI { return tempViewProvider; } + /** + * Close tempViewProvider. + *

+ * For test, avoid multiple SparkContexts. + */ + public static synchronized void closeTempViewProvider() { + if (tempViewProvider != null) { + tempViewProvider.close(); + tempViewProvider = null; + } + } + } diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java index 39e3767e2..975e89fc7 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java @@ -20,36 +20,55 @@ package org.apache.hudi.cli.commands; import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.exception.HoodieException; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; -import java.io.IOException; - /** * CLI command to query/delete temp views. */ @Component public class TempViewCommand implements CommandMarker { - private static final String EMPTY_STRING = ""; + public static final String QUERY_SUCCESS = "Query ran successfully!"; + public static final String QUERY_FAIL = "Query ran failed!"; + public static final String SHOW_SUCCESS = "Show all views name successfully!"; - @CliCommand(value = "temp_query", help = "query against created temp view") + @CliCommand(value = {"temp_query", "temp query"}, help = "query against created temp view") public String query( - @CliOption(key = {"sql"}, mandatory = true, help = "select query to run against view") final String sql) - throws IOException { + @CliOption(key = {"sql"}, mandatory = true, help = "select query to run against view") final String sql) { + + try { + HoodieCLI.getTempViewProvider().runQuery(sql); + return QUERY_SUCCESS; + } catch (HoodieException ex) { + return QUERY_FAIL; + } - HoodieCLI.getTempViewProvider().runQuery(sql); - return EMPTY_STRING; } - @CliCommand(value = "temp_delete", help = "Delete view name") - public String delete( - @CliOption(key = {"view"}, mandatory = true, help = "view name") final String tableName) - throws IOException { + @CliCommand(value = {"temps_show", "temps show"}, help = "Show all views name") + public String showAll() { - HoodieCLI.getTempViewProvider().deleteTable(tableName); - return EMPTY_STRING; + try { + HoodieCLI.getTempViewProvider().showAllViews(); + return SHOW_SUCCESS; + } catch (HoodieException ex) { + return "Show all views name failed!"; + } + } + + @CliCommand(value = {"temp_delete", "temp delete"}, help = "Delete view name") + public String delete( + @CliOption(key = {"view"}, mandatory = true, help = "view name") final String tableName) { + + try { + HoodieCLI.getTempViewProvider().deleteTable(tableName); + return String.format("Delete view %s successfully!", tableName); + } catch (HoodieException ex) { + return String.format("Delete view %s failed!", tableName); + } } } diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java index 39abcbb39..5e029cd05 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java @@ -101,6 +101,17 @@ public class SparkTempViewProvider implements TempViewProvider { } } + @Override + public void showAllViews() { + try { + sqlContext.sql("SHOW TABLES").show(Integer.MAX_VALUE, false); + } catch (Throwable ex) { + // log full stack trace and rethrow. Without this its difficult to debug failures, if any + LOG.error("unable to get all views ", ex); + throw new HoodieException(ex); + } + } + @Override public void deleteTable(String tableName) { try { @@ -112,6 +123,13 @@ public class SparkTempViewProvider implements TempViewProvider { } } + @Override + public void close() { + if (sqlContext != null) { + sqlContext.sparkSession().stop(); + } + } + private DataType getDataType(Comparable comparable) { if (comparable instanceof Integer) { return DataTypes.IntegerType; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/TempViewProvider.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/TempViewProvider.java index 1075fddc1..421ad5959 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/TempViewProvider.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/TempViewProvider.java @@ -18,12 +18,17 @@ package org.apache.hudi.cli.utils; +import java.io.Closeable; import java.util.List; -public interface TempViewProvider { +public interface TempViewProvider extends Closeable { void createOrReplace(String tableName, List headers, List> rows); void runQuery(String sqlText); + void showAllViews(); + void deleteTable(String tableName); + + void close(); } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTempViewCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTempViewCommand.java new file mode 100644 index 000000000..504f0d7ba --- /dev/null +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTempViewCommand.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.cli.commands; + +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.testutils.AbstractShellBaseIntegrationTest; +import org.apache.hudi.exception.HoodieException; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.shell.core.CommandResult; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestTempViewCommand extends AbstractShellBaseIntegrationTest { + + private String tableName = "test_table"; + + @BeforeEach + public void init() { + List> rows = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + rows.add(Arrays.asList(new Comparable[] {"c1", "c2", "c3"})); + } + HoodieCLI.getTempViewProvider().createOrReplace(tableName, Arrays.asList("t1", "t2", "t3"), rows); + } + + @AfterAll + public static void shutdown() { + if (HoodieCLI.getTempViewProvider() != null) { + HoodieCLI.closeTempViewProvider(); + } + } + + @Test + public void testQueryWithException() { + CommandResult cr = getShell().executeCommand(String.format("temp query --sql 'select * from %s'", "table_1")); + assertEquals(TempViewCommand.QUERY_FAIL, cr.getResult().toString()); + } + + @Test + public void testQuery() { + CommandResult cr = getShell().executeCommand(String.format("temp query --sql 'select * from %s'", tableName)); + assertEquals(TempViewCommand.QUERY_SUCCESS, cr.getResult().toString()); + } + + @Test + public void testShowAll() { + CommandResult cr = getShell().executeCommand("temps show"); + assertEquals(TempViewCommand.SHOW_SUCCESS, cr.getResult().toString()); + } + + @Test + public void testDelete() { + CommandResult cr = getShell().executeCommand(String.format("temp delete --view %s", tableName)); + assertTrue(cr.getResult().toString().endsWith("successfully!")); + + // after delete, we can not access table yet. + assertThrows(HoodieException.class, () -> HoodieCLI.getTempViewProvider().runQuery("select * from " + tableName)); + } +} diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellBaseIntegrationTest.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellBaseIntegrationTest.java new file mode 100644 index 000000000..e01656443 --- /dev/null +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellBaseIntegrationTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.cli.testutils; + +import org.apache.hudi.testutils.HoodieClientTestHarness; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.shell.Bootstrap; +import org.springframework.shell.core.JLineShellComponent; + +/** + * Class to start Bootstrap and JLineShellComponent. + */ +public class AbstractShellBaseIntegrationTest extends HoodieClientTestHarness { + + private static JLineShellComponent shell; + + @BeforeAll + public static void startup() { + Bootstrap bootstrap = new Bootstrap(); + shell = bootstrap.getJLineShellComponent(); + } + + @AfterAll + public static void shutdown() { + shell.stop(); + } + + @BeforeEach + public void setup() throws Exception { + initPath(); + } + + @AfterEach + public void teardown() throws Exception { + System.gc(); + } + + protected static JLineShellComponent getShell() { + return shell; + } + + /** + * Helper to prepare string for matching. + * @param str Input string. + * @return pruned string with non word characters removed. + */ + protected String removeNonWordAndStripSpace(String str) { + return str.replaceAll("[\\s]+", ",").replaceAll("[\\W]+", ","); + } +} diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java index 0c98135dc..a7cf85cf9 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java @@ -18,53 +18,23 @@ package org.apache.hudi.cli.testutils; -import org.apache.hudi.testutils.HoodieClientTestHarness; - -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.springframework.shell.Bootstrap; -import org.springframework.shell.core.JLineShellComponent; /** - * Class to start Bootstrap and JLineShellComponent. + * Class to initial resources for shell. */ -public abstract class AbstractShellIntegrationTest extends HoodieClientTestHarness { - - private static JLineShellComponent shell; - - @BeforeAll - public static void startup() { - Bootstrap bootstrap = new Bootstrap(); - shell = bootstrap.getJLineShellComponent(); - } - - @AfterAll - public static void shutdown() { - shell.stop(); - } +public abstract class AbstractShellIntegrationTest extends AbstractShellBaseIntegrationTest { + @Override @BeforeEach public void setup() throws Exception { initResources(); } + @Override @AfterEach public void teardown() throws Exception { cleanupResources(); } - - protected static JLineShellComponent getShell() { - return shell; - } - - /** - * Helper to prepare string for matching. - * @param str Input string. - * @return pruned string with non word characters removed. - */ - protected String removeNonWordAndStripSpace(String str) { - return str.replaceAll("[\\s]+", ",").replaceAll("[\\W]+", ","); - } }