From fcabc8fbca1670c9d50105495278ff147ccbdd4a Mon Sep 17 00:00:00 2001 From: hongdd Date: Sun, 14 Jun 2020 14:30:28 +0800 Subject: [PATCH] [HUDI-1019] Clean refresh command in CLI (#1725) --- .../hudi/cli/commands/CleansCommand.java | 6 --- .../hudi/cli/commands/CommitsCommand.java | 8 +-- .../hudi/cli/commands/SavepointsCommand.java | 12 ++--- .../hudi/cli/commands/TableCommand.java | 10 ++++ .../hudi/cli/commands/TestTableCommand.java | 52 +++++++++++++++++++ 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java index 218cf362a..4924eaacb 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java @@ -84,12 +84,6 @@ public class CleansCommand implements CommandMarker { return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows); } - @CliCommand(value = "cleans refresh", help = "Refresh the commits") - public String refreshCleans() { - HoodieCLI.refreshTableMetadata(); - return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " refreshed."; - } - @CliCommand(value = "clean showpartitions", help = "Show partition level details of a clean") public String showCleanPartitions(@CliOption(key = {"clean"}, help = "clean to show") final String instantTime, @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java index 4288c2abf..6670067d3 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java @@ -214,12 +214,6 @@ public class CommitsCommand implements CommandMarker { } } - @CliCommand(value = "commits refresh", help = "Refresh the commits") - public String refreshCommits() throws IOException { - HoodieCLI.refreshTableMetadata(); - return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " refreshed."; - } - @CliCommand(value = "commit rollback", help = "Rollback a commit") public String rollbackCommit(@CliOption(key = {"commit"}, help = "Commit to rollback") final String instantTime, @CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path") final String sparkPropertiesPath) @@ -238,7 +232,7 @@ public class CommitsCommand implements CommandMarker { InputStreamConsumer.captureOutput(process); int exitCode = process.waitFor(); // Refresh the current - refreshCommits(); + HoodieCLI.refreshTableMetadata(); if (exitCode != 0) { return "Commit " + instantTime + " failed to roll back"; } diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java index 314ed00c5..6d8155295 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java @@ -89,7 +89,7 @@ public class SavepointsCommand implements CommandMarker { InputStreamConsumer.captureOutput(process); int exitCode = process.waitFor(); // Refresh the current - refreshMetaClient(); + HoodieCLI.refreshTableMetadata(); if (exitCode != 0) { return String.format("Failed: Could not create savepoint \"%s\".", commitTime); } @@ -123,19 +123,13 @@ public class SavepointsCommand implements CommandMarker { InputStreamConsumer.captureOutput(process); int exitCode = process.waitFor(); // Refresh the current - refreshMetaClient(); + HoodieCLI.refreshTableMetadata(); if (exitCode != 0) { return String.format("Savepoint \"%s\" failed to roll back", instantTime); } return String.format("Savepoint \"%s\" rolled back", instantTime); } - @CliCommand(value = "savepoints refresh", help = "Refresh the savepoints") - public String refreshMetaClient() { - HoodieCLI.refreshTableMetadata(); - return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " refreshed."; - } - @CliCommand(value = "savepoint delete", help = "Delete the savepoint") public String deleteSavepoint(@CliOption(key = {"commit"}, help = "Delete a savepoint") final String instantTime, @CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path") final String sparkPropertiesPath, @@ -161,7 +155,7 @@ public class SavepointsCommand implements CommandMarker { InputStreamConsumer.captureOutput(process); int exitCode = process.waitFor(); // Refresh the current - refreshMetaClient(); + HoodieCLI.refreshTableMetadata(); if (exitCode != 0) { return String.format("Failed: Could not delete savepoint \"%s\".", instantTime); } diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java index da3d57a4f..78ae35e19 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java @@ -131,4 +131,14 @@ public class TableCommand implements CommandMarker { }); return HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); } + + /** + * Refresh table metadata. + */ + @CliCommand(value = {"refresh", "metadata refresh", "commits refresh", "cleans refresh", "savepoints refresh"}, + help = "Refresh table metadata") + public String refreshMetadata() { + HoodieCLI.refreshTableMetadata(); + return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " refreshed."; + } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java index 006cd45b7..411ca4db0 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java @@ -18,17 +18,24 @@ package org.apache.hudi.cli.commands; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; import static org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -140,4 +147,49 @@ public class TestTableCommand extends AbstractShellIntegrationTest { assertTrue(cr.getResult().toString().contains(metaPath)); assertTrue(cr.getResult().toString().contains("COPY_ON_WRITE")); } + + /** + * Test case of command 'refresh'. + */ + @Test + public void testRefresh() throws IOException { + List refreshCommands = Arrays.asList("refresh", "metadata refresh", + "commits refresh", "cleans refresh", "savepoints refresh"); + for (String command: refreshCommands) { + testRefreshCommand(command); + } + } + + private void testRefreshCommand(String command) throws IOException { + // clean table matedata + FileSystem fs = FileSystem.get(jsc.hadoopConfiguration()); + fs.delete(new Path(tablePath + File.separator + HoodieTableMetaClient.METAFOLDER_NAME), true); + + // Create table + assertTrue(prepareTable()); + + HoodieTimeline timeline = + HoodieCLI.getTableMetaClient().getActiveTimeline().getCommitTimeline().filterCompletedInstants(); + assertEquals(0, timeline.countInstants(), "There should have no instant at first"); + + // generate four savepoints + for (int i = 100; i < 104; i++) { + String instantTime = String.valueOf(i); + HoodieTestDataGenerator.createCommitFile(tablePath, instantTime, jsc.hadoopConfiguration()); + } + + // Before refresh, no instant + timeline = + HoodieCLI.getTableMetaClient().getActiveTimeline().getCommitTimeline().filterCompletedInstants(); + assertEquals(0, timeline.countInstants(), "there should have no instant"); + + CommandResult cr = getShell().executeCommand(command); + assertTrue(cr.isSuccess()); + + timeline = + HoodieCLI.getTableMetaClient().getActiveTimeline().getCommitTimeline().filterCompletedInstants(); + + // After refresh, there are 4 instants + assertEquals(4, timeline.countInstants(), "there should have 4 instants"); + } }