[HUDI-416] Improve hint information for cli (#1110)
This commit is contained in:
@@ -40,7 +40,7 @@ public class HoodieCLI {
|
|||||||
public static FileSystem fs;
|
public static FileSystem fs;
|
||||||
public static CLIState state = CLIState.INIT;
|
public static CLIState state = CLIState.INIT;
|
||||||
public static String basePath;
|
public static String basePath;
|
||||||
public static HoodieTableMetaClient tableMetadata;
|
protected static HoodieTableMetaClient tableMetadata;
|
||||||
public static HoodieTableMetaClient syncTableMetadata;
|
public static HoodieTableMetaClient syncTableMetadata;
|
||||||
public static TimelineLayoutVersion layoutVersion;
|
public static TimelineLayoutVersion layoutVersion;
|
||||||
|
|
||||||
@@ -92,4 +92,17 @@ public class HoodieCLI {
|
|||||||
setLayoutVersion(layoutVersion);
|
setLayoutVersion(layoutVersion);
|
||||||
refreshTableMetadata();
|
refreshTableMetadata();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get tableMetadata, throw NullPointerException when it is null.
|
||||||
|
*
|
||||||
|
* @return tableMetadata which is instance of HoodieTableMetaClient
|
||||||
|
*/
|
||||||
|
public static HoodieTableMetaClient getTableMetaClient() {
|
||||||
|
if (tableMetadata == null) {
|
||||||
|
throw new NullPointerException("There is no hudi dataset. Please use connect command to set dataset first");
|
||||||
|
}
|
||||||
|
return tableMetadata;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ import org.apache.avro.specific.SpecificData;
|
|||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -53,11 +52,6 @@ import java.util.stream.Collectors;
|
|||||||
@Component
|
@Component
|
||||||
public class ArchivedCommitsCommand implements CommandMarker {
|
public class ArchivedCommitsCommand implements CommandMarker {
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"show archived commits"})
|
|
||||||
public boolean isShowArchivedCommitAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "show archived commit stats", help = "Read commits from archived files and show details")
|
@CliCommand(value = "show archived commit stats", help = "Read commits from archived files and show details")
|
||||||
public String showArchivedCommits(
|
public String showArchivedCommits(
|
||||||
@CliOption(key = {"archiveFolderPattern"}, help = "Archive Folder", unspecifiedDefaultValue = "") String folder,
|
@CliOption(key = {"archiveFolderPattern"}, help = "Archive Folder", unspecifiedDefaultValue = "") String folder,
|
||||||
@@ -68,7 +62,7 @@ public class ArchivedCommitsCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
System.out.println("===============> Showing only " + limit + " archived commits <===============");
|
System.out.println("===============> Showing only " + limit + " archived commits <===============");
|
||||||
String basePath = HoodieCLI.tableMetadata.getBasePath();
|
String basePath = HoodieCLI.getTableMetaClient().getBasePath();
|
||||||
Path archivePath = new Path(basePath + "/.hoodie/.commits_.archive*");
|
Path archivePath = new Path(basePath + "/.hoodie/.commits_.archive*");
|
||||||
if (folder != null && !folder.isEmpty()) {
|
if (folder != null && !folder.isEmpty()) {
|
||||||
archivePath = new Path(basePath + "/.hoodie/" + folder);
|
archivePath = new Path(basePath + "/.hoodie/" + folder);
|
||||||
@@ -146,7 +140,7 @@ public class ArchivedCommitsCommand implements CommandMarker {
|
|||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
System.out.println("===============> Showing only " + limit + " archived commits <===============");
|
System.out.println("===============> Showing only " + limit + " archived commits <===============");
|
||||||
String basePath = HoodieCLI.tableMetadata.getBasePath();
|
String basePath = HoodieCLI.getTableMetaClient().getBasePath();
|
||||||
FileStatus[] fsStatuses =
|
FileStatus[] fsStatuses =
|
||||||
FSUtils.getFs(basePath, HoodieCLI.conf).globStatus(new Path(basePath + "/.hoodie/.commits_.archive*"));
|
FSUtils.getFs(basePath, HoodieCLI.conf).globStatus(new Path(basePath + "/.hoodie/.commits_.archive*"));
|
||||||
List<Comparable[]> allCommits = new ArrayList<>();
|
List<Comparable[]> allCommits = new ArrayList<>();
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
|
|||||||
import org.apache.hudi.common.util.AvroUtils;
|
import org.apache.hudi.common.util.AvroUtils;
|
||||||
|
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -47,21 +46,6 @@ import java.util.stream.Collectors;
|
|||||||
@Component
|
@Component
|
||||||
public class CleansCommand implements CommandMarker {
|
public class CleansCommand implements CommandMarker {
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"cleans show"})
|
|
||||||
public boolean isShowAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"cleans refresh"})
|
|
||||||
public boolean isRefreshAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"clean showpartitions"})
|
|
||||||
public boolean isCommitShowAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "cleans show", help = "Show the cleans")
|
@CliCommand(value = "cleans show", help = "Show the cleans")
|
||||||
public String showCleans(
|
public String showCleans(
|
||||||
@CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
|
@CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
|
||||||
@@ -71,7 +55,7 @@ public class CleansCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCleanerTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCleanerTimeline().filterCompletedInstants();
|
||||||
List<HoodieInstant> cleans = timeline.getReverseOrderedInstants().collect(Collectors.toList());
|
List<HoodieInstant> cleans = timeline.getReverseOrderedInstants().collect(Collectors.toList());
|
||||||
List<Comparable[]> rows = new ArrayList<>();
|
List<Comparable[]> rows = new ArrayList<>();
|
||||||
@@ -92,7 +76,7 @@ public class CleansCommand implements CommandMarker {
|
|||||||
@CliCommand(value = "cleans refresh", help = "Refresh the commits")
|
@CliCommand(value = "cleans refresh", help = "Refresh the commits")
|
||||||
public String refreshCleans() throws IOException {
|
public String refreshCleans() throws IOException {
|
||||||
HoodieCLI.refreshTableMetadata();
|
HoodieCLI.refreshTableMetadata();
|
||||||
return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " refreshed.";
|
return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " refreshed.";
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliCommand(value = "clean showpartitions", help = "Show partition level details of a clean")
|
@CliCommand(value = "clean showpartitions", help = "Show partition level details of a clean")
|
||||||
@@ -104,7 +88,7 @@ public class CleansCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCleanerTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCleanerTimeline().filterCompletedInstants();
|
||||||
HoodieInstant cleanInstant = new HoodieInstant(false, HoodieTimeline.CLEAN_ACTION, commitTime);
|
HoodieInstant cleanInstant = new HoodieInstant(false, HoodieTimeline.CLEAN_ACTION, commitTime);
|
||||||
|
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ import org.apache.hudi.common.util.NumericUtils;
|
|||||||
|
|
||||||
import org.apache.spark.launcher.SparkLauncher;
|
import org.apache.spark.launcher.SparkLauncher;
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -52,26 +51,6 @@ import java.util.stream.Collectors;
|
|||||||
@Component
|
@Component
|
||||||
public class CommitsCommand implements CommandMarker {
|
public class CommitsCommand implements CommandMarker {
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"commits show"})
|
|
||||||
public boolean isShowAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"commits refresh"})
|
|
||||||
public boolean isRefreshAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"commit rollback"})
|
|
||||||
public boolean isRollbackAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"commit show"})
|
|
||||||
public boolean isCommitShowAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "commits show", help = "Show the commits")
|
@CliCommand(value = "commits show", help = "Show the commits")
|
||||||
public String showCommits(
|
public String showCommits(
|
||||||
@CliOption(key = {"limit"}, mandatory = false, help = "Limit commits",
|
@CliOption(key = {"limit"}, mandatory = false, help = "Limit commits",
|
||||||
@@ -82,7 +61,7 @@ public class CommitsCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
||||||
List<HoodieInstant> commits = timeline.getReverseOrderedInstants().collect(Collectors.toList());
|
List<HoodieInstant> commits = timeline.getReverseOrderedInstants().collect(Collectors.toList());
|
||||||
List<Comparable[]> rows = new ArrayList<>();
|
List<Comparable[]> rows = new ArrayList<>();
|
||||||
@@ -111,14 +90,14 @@ public class CommitsCommand implements CommandMarker {
|
|||||||
@CliCommand(value = "commits refresh", help = "Refresh the commits")
|
@CliCommand(value = "commits refresh", help = "Refresh the commits")
|
||||||
public String refreshCommits() throws IOException {
|
public String refreshCommits() throws IOException {
|
||||||
HoodieCLI.refreshTableMetadata();
|
HoodieCLI.refreshTableMetadata();
|
||||||
return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " refreshed.";
|
return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " refreshed.";
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliCommand(value = "commit rollback", help = "Rollback a commit")
|
@CliCommand(value = "commit rollback", help = "Rollback a commit")
|
||||||
public String rollbackCommit(@CliOption(key = {"commit"}, help = "Commit to rollback") final String commitTime,
|
public String rollbackCommit(@CliOption(key = {"commit"}, help = "Commit to rollback") final String commitTime,
|
||||||
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path") final String sparkPropertiesPath)
|
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path") final String sparkPropertiesPath)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
||||||
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
||||||
|
|
||||||
@@ -128,7 +107,7 @@ public class CommitsCommand implements CommandMarker {
|
|||||||
|
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkMain.SparkCommand.ROLLBACK.toString(), commitTime,
|
sparkLauncher.addAppArgs(SparkMain.SparkCommand.ROLLBACK.toString(), commitTime,
|
||||||
HoodieCLI.tableMetadata.getBasePath());
|
HoodieCLI.getTableMetaClient().getBasePath());
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
InputStreamConsumer.captureOutput(process);
|
InputStreamConsumer.captureOutput(process);
|
||||||
int exitCode = process.waitFor();
|
int exitCode = process.waitFor();
|
||||||
@@ -149,7 +128,7 @@ public class CommitsCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
||||||
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
||||||
|
|
||||||
@@ -205,7 +184,7 @@ public class CommitsCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
|
||||||
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
||||||
|
|
||||||
@@ -232,18 +211,13 @@ public class CommitsCommand implements CommandMarker {
|
|||||||
return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows);
|
return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"commits compare"})
|
|
||||||
public boolean isCompareCommitsAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "commits compare", help = "Compare commits with another Hoodie dataset")
|
@CliCommand(value = "commits compare", help = "Compare commits with another Hoodie dataset")
|
||||||
public String compareCommits(@CliOption(key = {"path"}, help = "Path of the dataset to compare to") final String path)
|
public String compareCommits(@CliOption(key = {"path"}, help = "Path of the dataset to compare to") final String path)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
|
||||||
|
HoodieTableMetaClient source = HoodieCLI.getTableMetaClient();
|
||||||
HoodieTableMetaClient target = new HoodieTableMetaClient(HoodieCLI.conf, path);
|
HoodieTableMetaClient target = new HoodieTableMetaClient(HoodieCLI.conf, path);
|
||||||
HoodieTimeline targetTimeline = target.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
|
HoodieTimeline targetTimeline = target.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
|
||||||
HoodieTableMetaClient source = HoodieCLI.tableMetadata;
|
|
||||||
HoodieTimeline sourceTimeline = source.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
|
HoodieTimeline sourceTimeline = source.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
|
||||||
String targetLatestCommit =
|
String targetLatestCommit =
|
||||||
targetTimeline.getInstants().iterator().hasNext() ? "0" : targetTimeline.lastInstant().get().getTimestamp();
|
targetTimeline.getInstants().iterator().hasNext() ? "0" : targetTimeline.lastInstant().get().getTimestamp();
|
||||||
@@ -265,17 +239,12 @@ public class CommitsCommand implements CommandMarker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"commits sync"})
|
|
||||||
public boolean isSyncCommitsAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "commits sync", help = "Compare commits with another Hoodie dataset")
|
@CliCommand(value = "commits sync", help = "Compare commits with another Hoodie dataset")
|
||||||
public String syncCommits(@CliOption(key = {"path"}, help = "Path of the dataset to compare to") final String path)
|
public String syncCommits(@CliOption(key = {"path"}, help = "Path of the dataset to compare to") final String path)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
HoodieCLI.syncTableMetadata = new HoodieTableMetaClient(HoodieCLI.conf, path);
|
HoodieCLI.syncTableMetadata = new HoodieTableMetaClient(HoodieCLI.conf, path);
|
||||||
HoodieCLI.state = HoodieCLI.CLIState.SYNC;
|
HoodieCLI.state = HoodieCLI.CLIState.SYNC;
|
||||||
return "Load sync state between " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " and "
|
return "Load sync state between " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " and "
|
||||||
+ HoodieCLI.syncTableMetadata.getTableConfig().getTableName();
|
+ HoodieCLI.syncTableMetadata.getTableConfig().getTableName();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -29,12 +29,14 @@ import org.apache.hudi.cli.commands.SparkMain.SparkCommand;
|
|||||||
import org.apache.hudi.cli.utils.InputStreamConsumer;
|
import org.apache.hudi.cli.utils.InputStreamConsumer;
|
||||||
import org.apache.hudi.cli.utils.SparkUtil;
|
import org.apache.hudi.cli.utils.SparkUtil;
|
||||||
import org.apache.hudi.common.model.HoodieTableType;
|
import org.apache.hudi.common.model.HoodieTableType;
|
||||||
|
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
||||||
import org.apache.hudi.common.table.HoodieTimeline;
|
import org.apache.hudi.common.table.HoodieTimeline;
|
||||||
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
|
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
|
||||||
import org.apache.hudi.common.table.timeline.HoodieInstant;
|
import org.apache.hudi.common.table.timeline.HoodieInstant;
|
||||||
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
|
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
|
||||||
import org.apache.hudi.common.util.AvroUtils;
|
import org.apache.hudi.common.util.AvroUtils;
|
||||||
import org.apache.hudi.common.util.Option;
|
import org.apache.hudi.common.util.Option;
|
||||||
|
import org.apache.hudi.exception.HoodieException;
|
||||||
import org.apache.hudi.exception.HoodieIOException;
|
import org.apache.hudi.exception.HoodieIOException;
|
||||||
|
|
||||||
import org.apache.hadoop.fs.FSDataInputStream;
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
@@ -45,7 +47,6 @@ import org.apache.log4j.Logger;
|
|||||||
import org.apache.spark.launcher.SparkLauncher;
|
import org.apache.spark.launcher.SparkLauncher;
|
||||||
import org.apache.spark.util.Utils;
|
import org.apache.spark.util.Utils;
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -71,10 +72,12 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
|
|
||||||
private static final String TMP_DIR = "/tmp/";
|
private static final String TMP_DIR = "/tmp/";
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"compactions show all", "compaction show", "compaction run", "compaction schedule"})
|
private HoodieTableMetaClient checkAndGetMetaClient() {
|
||||||
public boolean isAvailable() {
|
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
|
||||||
return (HoodieCLI.tableMetadata != null)
|
if (client.getTableType() != HoodieTableType.MERGE_ON_READ) {
|
||||||
&& (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ);
|
throw new HoodieException("Compactions can only be run for table type : MERGE_ON_READ");
|
||||||
|
}
|
||||||
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliCommand(value = "compactions show all", help = "Shows all compactions that are in active timeline")
|
@CliCommand(value = "compactions show all", help = "Shows all compactions that are in active timeline")
|
||||||
@@ -88,7 +91,8 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
|
HoodieActiveTimeline activeTimeline = client.getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitsAndCompactionTimeline();
|
HoodieTimeline timeline = activeTimeline.getCommitsAndCompactionTimeline();
|
||||||
HoodieTimeline commitTimeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
HoodieTimeline commitTimeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
||||||
Set<String> committed = commitTimeline.getInstants().map(HoodieInstant::getTimestamp).collect(Collectors.toSet());
|
Set<String> committed = commitTimeline.getInstants().map(HoodieInstant::getTimestamp).collect(Collectors.toSet());
|
||||||
@@ -148,7 +152,8 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
|
HoodieActiveTimeline activeTimeline = client.getActiveTimeline();
|
||||||
HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
|
HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
|
||||||
activeTimeline.readPlanAsBytes(
|
activeTimeline.readPlanAsBytes(
|
||||||
HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get());
|
HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get());
|
||||||
@@ -171,18 +176,18 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliCommand(value = "compaction schedule", help = "Schedule Compaction")
|
@CliCommand(value = "compaction schedule", help = "Schedule Compaction")
|
||||||
public String scheduleCompact(@CliOption(key = "sparkMemory", unspecifiedDefaultValue = "1G",
|
public String scheduleCompact(@CliOption(key = "sparkMemory", unspecifiedDefaultValue = "1G",
|
||||||
help = "Spark executor memory") final String sparkMemory) throws Exception {
|
help = "Spark executor memory") final String sparkMemory) throws Exception {
|
||||||
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
boolean initialized = HoodieCLI.initConf();
|
boolean initialized = HoodieCLI.initConf();
|
||||||
HoodieCLI.initFS(initialized);
|
HoodieCLI.initFS(initialized);
|
||||||
|
|
||||||
// First get a compaction instant time and pass it to spark launcher for scheduling compaction
|
// First get a compaction instant time and pass it to spark launcher for scheduling compaction
|
||||||
String compactionInstantTime = HoodieActiveTimeline.createNewInstantTime();
|
String compactionInstantTime = HoodieActiveTimeline.createNewInstantTime();
|
||||||
|
|
||||||
if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) {
|
|
||||||
String sparkPropertiesPath =
|
String sparkPropertiesPath =
|
||||||
Utils.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
Utils.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkCommand.COMPACT_SCHEDULE.toString(), HoodieCLI.tableMetadata.getBasePath(),
|
sparkLauncher.addAppArgs(SparkCommand.COMPACT_SCHEDULE.toString(), client.getBasePath(),
|
||||||
HoodieCLI.tableMetadata.getTableConfig().getTableName(), compactionInstantTime, sparkMemory);
|
client.getTableConfig().getTableName(), compactionInstantTime, sparkMemory);
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
InputStreamConsumer.captureOutput(process);
|
InputStreamConsumer.captureOutput(process);
|
||||||
int exitCode = process.waitFor();
|
int exitCode = process.waitFor();
|
||||||
@@ -190,9 +195,6 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
return "Failed to run compaction for " + compactionInstantTime;
|
return "Failed to run compaction for " + compactionInstantTime;
|
||||||
}
|
}
|
||||||
return "Compaction successfully completed for " + compactionInstantTime;
|
return "Compaction successfully completed for " + compactionInstantTime;
|
||||||
} else {
|
|
||||||
throw new Exception("Compactions can only be run for table type : MERGE_ON_READ");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliCommand(value = "compaction run", help = "Run Compaction for given instant time")
|
@CliCommand(value = "compaction run", help = "Run Compaction for given instant time")
|
||||||
@@ -207,14 +209,14 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliOption(key = "compactionInstant", mandatory = false,
|
@CliOption(key = "compactionInstant", mandatory = false,
|
||||||
help = "Base path for the target hoodie dataset") String compactionInstantTime)
|
help = "Base path for the target hoodie dataset") String compactionInstantTime)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
boolean initialized = HoodieCLI.initConf();
|
boolean initialized = HoodieCLI.initConf();
|
||||||
HoodieCLI.initFS(initialized);
|
HoodieCLI.initFS(initialized);
|
||||||
|
|
||||||
if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) {
|
|
||||||
if (null == compactionInstantTime) {
|
if (null == compactionInstantTime) {
|
||||||
// pick outstanding one with lowest timestamp
|
// pick outstanding one with lowest timestamp
|
||||||
Option<String> firstPendingInstant =
|
Option<String> firstPendingInstant =
|
||||||
HoodieCLI.tableMetadata.reloadActiveTimeline().filterCompletedAndCompactionInstants()
|
client.reloadActiveTimeline().filterCompletedAndCompactionInstants()
|
||||||
.filter(instant -> instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)).firstInstant()
|
.filter(instant -> instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)).firstInstant()
|
||||||
.map(HoodieInstant::getTimestamp);
|
.map(HoodieInstant::getTimestamp);
|
||||||
if (!firstPendingInstant.isPresent()) {
|
if (!firstPendingInstant.isPresent()) {
|
||||||
@@ -226,8 +228,8 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
String sparkPropertiesPath =
|
String sparkPropertiesPath =
|
||||||
Utils.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
Utils.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkCommand.COMPACT_RUN.toString(), HoodieCLI.tableMetadata.getBasePath(),
|
sparkLauncher.addAppArgs(SparkCommand.COMPACT_RUN.toString(), client.getBasePath(),
|
||||||
HoodieCLI.tableMetadata.getTableConfig().getTableName(), compactionInstantTime, parallelism, schemaFilePath,
|
client.getTableConfig().getTableName(), compactionInstantTime, parallelism, schemaFilePath,
|
||||||
sparkMemory, retry);
|
sparkMemory, retry);
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
InputStreamConsumer.captureOutput(process);
|
InputStreamConsumer.captureOutput(process);
|
||||||
@@ -236,9 +238,6 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
return "Failed to run compaction for " + compactionInstantTime;
|
return "Failed to run compaction for " + compactionInstantTime;
|
||||||
}
|
}
|
||||||
return "Compaction successfully completed for " + compactionInstantTime;
|
return "Compaction successfully completed for " + compactionInstantTime;
|
||||||
} else {
|
|
||||||
throw new Exception("Compactions can only be run for table type : MERGE_ON_READ");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String getTmpSerializerFile() {
|
private static String getTmpSerializerFile() {
|
||||||
@@ -271,18 +270,18 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
||||||
unspecifiedDefaultValue = "false") boolean headerOnly)
|
unspecifiedDefaultValue = "false") boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
boolean initialized = HoodieCLI.initConf();
|
boolean initialized = HoodieCLI.initConf();
|
||||||
HoodieCLI.initFS(initialized);
|
HoodieCLI.initFS(initialized);
|
||||||
|
|
||||||
String outputPathStr = getTmpSerializerFile();
|
String outputPathStr = getTmpSerializerFile();
|
||||||
Path outputPath = new Path(outputPathStr);
|
Path outputPath = new Path(outputPathStr);
|
||||||
String output = null;
|
String output = null;
|
||||||
if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) {
|
|
||||||
try {
|
try {
|
||||||
String sparkPropertiesPath = Utils
|
String sparkPropertiesPath = Utils
|
||||||
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkCommand.COMPACT_VALIDATE.toString(), HoodieCLI.tableMetadata.getBasePath(),
|
sparkLauncher.addAppArgs(SparkCommand.COMPACT_VALIDATE.toString(), client.getBasePath(),
|
||||||
compactionInstant, outputPathStr, parallelism, master, sparkMemory);
|
compactionInstant, outputPathStr, parallelism, master, sparkMemory);
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
InputStreamConsumer.captureOutput(process);
|
InputStreamConsumer.captureOutput(process);
|
||||||
@@ -316,9 +315,6 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return output;
|
return output;
|
||||||
} else {
|
|
||||||
throw new Exception("Compactions can only be run for table type : MERGE_ON_READ");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliCommand(value = "compaction unschedule", help = "Unschedule Compaction")
|
@CliCommand(value = "compaction unschedule", help = "Unschedule Compaction")
|
||||||
@@ -335,18 +331,18 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
||||||
unspecifiedDefaultValue = "false") boolean headerOnly)
|
unspecifiedDefaultValue = "false") boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
boolean initialized = HoodieCLI.initConf();
|
boolean initialized = HoodieCLI.initConf();
|
||||||
HoodieCLI.initFS(initialized);
|
HoodieCLI.initFS(initialized);
|
||||||
|
|
||||||
String outputPathStr = getTmpSerializerFile();
|
String outputPathStr = getTmpSerializerFile();
|
||||||
Path outputPath = new Path(outputPathStr);
|
Path outputPath = new Path(outputPathStr);
|
||||||
String output = "";
|
String output = "";
|
||||||
if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) {
|
|
||||||
try {
|
try {
|
||||||
String sparkPropertiesPath = Utils
|
String sparkPropertiesPath = Utils
|
||||||
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkCommand.COMPACT_UNSCHEDULE_PLAN.toString(), HoodieCLI.tableMetadata.getBasePath(),
|
sparkLauncher.addAppArgs(SparkCommand.COMPACT_UNSCHEDULE_PLAN.toString(), client.getBasePath(),
|
||||||
compactionInstant, outputPathStr, parallelism, master, sparkMemory, Boolean.valueOf(skipV).toString(),
|
compactionInstant, outputPathStr, parallelism, master, sparkMemory, Boolean.valueOf(skipV).toString(),
|
||||||
Boolean.valueOf(dryRun).toString());
|
Boolean.valueOf(dryRun).toString());
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
@@ -365,9 +361,6 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return output;
|
return output;
|
||||||
} else {
|
|
||||||
throw new Exception("Compactions can only be run for table type : MERGE_ON_READ");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliCommand(value = "compaction unscheduleFileId", help = "UnSchedule Compaction for a fileId")
|
@CliCommand(value = "compaction unscheduleFileId", help = "UnSchedule Compaction for a fileId")
|
||||||
@@ -382,18 +375,18 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") boolean descending,
|
@CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") boolean descending,
|
||||||
@CliOption(key = {"headeronly"}, help = "Header Only", unspecifiedDefaultValue = "false") boolean headerOnly)
|
@CliOption(key = {"headeronly"}, help = "Header Only", unspecifiedDefaultValue = "false") boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
boolean initialized = HoodieCLI.initConf();
|
boolean initialized = HoodieCLI.initConf();
|
||||||
HoodieCLI.initFS(initialized);
|
HoodieCLI.initFS(initialized);
|
||||||
|
|
||||||
String outputPathStr = getTmpSerializerFile();
|
String outputPathStr = getTmpSerializerFile();
|
||||||
Path outputPath = new Path(outputPathStr);
|
Path outputPath = new Path(outputPathStr);
|
||||||
String output = "";
|
String output = "";
|
||||||
if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) {
|
|
||||||
try {
|
try {
|
||||||
String sparkPropertiesPath = Utils
|
String sparkPropertiesPath = Utils
|
||||||
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkCommand.COMPACT_UNSCHEDULE_FILE.toString(), HoodieCLI.tableMetadata.getBasePath(),
|
sparkLauncher.addAppArgs(SparkCommand.COMPACT_UNSCHEDULE_FILE.toString(), client.getBasePath(),
|
||||||
fileId, outputPathStr, "1", master, sparkMemory, Boolean.valueOf(skipV).toString(),
|
fileId, outputPathStr, "1", master, sparkMemory, Boolean.valueOf(skipV).toString(),
|
||||||
Boolean.valueOf(dryRun).toString());
|
Boolean.valueOf(dryRun).toString());
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
@@ -412,9 +405,6 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return output;
|
return output;
|
||||||
} else {
|
|
||||||
throw new Exception("Compactions can only be run for table type : MERGE_ON_READ");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliCommand(value = "compaction repair", help = "Renames the files to make them consistent with the timeline as "
|
@CliCommand(value = "compaction repair", help = "Renames the files to make them consistent with the timeline as "
|
||||||
@@ -431,17 +421,18 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
||||||
unspecifiedDefaultValue = "false") boolean headerOnly)
|
unspecifiedDefaultValue = "false") boolean headerOnly)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
HoodieTableMetaClient client = checkAndGetMetaClient();
|
||||||
boolean initialized = HoodieCLI.initConf();
|
boolean initialized = HoodieCLI.initConf();
|
||||||
HoodieCLI.initFS(initialized);
|
HoodieCLI.initFS(initialized);
|
||||||
|
|
||||||
String outputPathStr = getTmpSerializerFile();
|
String outputPathStr = getTmpSerializerFile();
|
||||||
Path outputPath = new Path(outputPathStr);
|
Path outputPath = new Path(outputPathStr);
|
||||||
String output = "";
|
String output = "";
|
||||||
if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) {
|
|
||||||
try {
|
try {
|
||||||
String sparkPropertiesPath = Utils
|
String sparkPropertiesPath = Utils
|
||||||
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkCommand.COMPACT_REPAIR.toString(), HoodieCLI.tableMetadata.getBasePath(),
|
sparkLauncher.addAppArgs(SparkCommand.COMPACT_REPAIR.toString(), client.getBasePath(),
|
||||||
compactionInstant, outputPathStr, parallelism, master, sparkMemory, Boolean.valueOf(dryRun).toString());
|
compactionInstant, outputPathStr, parallelism, master, sparkMemory, Boolean.valueOf(dryRun).toString());
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
InputStreamConsumer.captureOutput(process);
|
InputStreamConsumer.captureOutput(process);
|
||||||
@@ -458,9 +449,6 @@ public class CompactionCommand implements CommandMarker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return output;
|
return output;
|
||||||
} else {
|
|
||||||
throw new Exception("Compactions can only be run for table type : MERGE_ON_READ");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getRenamesToBePrinted(List<RenameOpResult> res, Integer limit, String sortByField, boolean descending,
|
private String getRenamesToBePrinted(List<RenameOpResult> res, Integer limit, String sortByField, boolean descending,
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ import org.apache.hudi.common.util.ConsistencyGuardConfig;
|
|||||||
import org.apache.hudi.exception.DatasetNotFoundException;
|
import org.apache.hudi.exception.DatasetNotFoundException;
|
||||||
|
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -69,7 +68,7 @@ public class DatasetsCommand implements CommandMarker {
|
|||||||
HoodieCLI.connectTo(path, layoutVersion);
|
HoodieCLI.connectTo(path, layoutVersion);
|
||||||
HoodieCLI.initFS(true);
|
HoodieCLI.initFS(true);
|
||||||
HoodieCLI.state = HoodieCLI.CLIState.DATASET;
|
HoodieCLI.state = HoodieCLI.CLIState.DATASET;
|
||||||
return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " loaded";
|
return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " loaded";
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -116,22 +115,18 @@ public class DatasetsCommand implements CommandMarker {
|
|||||||
return connect(path, layoutVersion, false, 0, 0, 0);
|
return connect(path, layoutVersion, false, 0, 0, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"desc"})
|
|
||||||
public boolean isDescAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Describes table properties.
|
* Describes table properties.
|
||||||
*/
|
*/
|
||||||
@CliCommand(value = "desc", help = "Describle Hoodie Table properties")
|
@CliCommand(value = "desc", help = "Describle Hoodie Table properties")
|
||||||
public String descTable() {
|
public String descTable() {
|
||||||
|
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
|
||||||
TableHeader header = new TableHeader().addTableHeaderField("Property").addTableHeaderField("Value");
|
TableHeader header = new TableHeader().addTableHeaderField("Property").addTableHeaderField("Value");
|
||||||
List<Comparable[]> rows = new ArrayList<>();
|
List<Comparable[]> rows = new ArrayList<>();
|
||||||
rows.add(new Comparable[] {"basePath", HoodieCLI.tableMetadata.getBasePath()});
|
rows.add(new Comparable[] {"basePath", client.getBasePath()});
|
||||||
rows.add(new Comparable[] {"metaPath", HoodieCLI.tableMetadata.getMetaPath()});
|
rows.add(new Comparable[] {"metaPath", client.getMetaPath()});
|
||||||
rows.add(new Comparable[] {"fileSystem", HoodieCLI.tableMetadata.getFs().getScheme()});
|
rows.add(new Comparable[] {"fileSystem", client.getFs().getScheme()});
|
||||||
HoodieCLI.tableMetadata.getTableConfig().getProps().entrySet().forEach(e -> {
|
client.getTableConfig().getProps().entrySet().forEach(e -> {
|
||||||
rows.add(new Comparable[] {e.getKey(), e.getValue()});
|
rows.add(new Comparable[] {e.getKey(), e.getValue()});
|
||||||
});
|
});
|
||||||
return HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
|
return HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
|
||||||
|
|||||||
@@ -141,7 +141,7 @@ public class FileSystemViewCommand implements CommandMarker {
|
|||||||
fileSliceStream = fsView.getLatestFileSlices(partition);
|
fileSliceStream = fsView.getLatestFileSlices(partition);
|
||||||
} else {
|
} else {
|
||||||
if (maxInstant.isEmpty()) {
|
if (maxInstant.isEmpty()) {
|
||||||
maxInstant = HoodieCLI.tableMetadata.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()
|
maxInstant = HoodieCLI.getTableMetaClient().getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()
|
||||||
.get().getTimestamp();
|
.get().getTimestamp();
|
||||||
}
|
}
|
||||||
fileSliceStream = fsView.getLatestMergedFileSlicesBeforeOrOn(partition, maxInstant);
|
fileSliceStream = fsView.getLatestMergedFileSlicesBeforeOrOn(partition, maxInstant);
|
||||||
@@ -224,10 +224,11 @@ public class FileSystemViewCommand implements CommandMarker {
|
|||||||
*/
|
*/
|
||||||
private HoodieTableFileSystemView buildFileSystemView(String globRegex, String maxInstant, boolean readOptimizedOnly,
|
private HoodieTableFileSystemView buildFileSystemView(String globRegex, String maxInstant, boolean readOptimizedOnly,
|
||||||
boolean includeMaxInstant, boolean includeInflight, boolean excludeCompaction) throws IOException {
|
boolean includeMaxInstant, boolean includeInflight, boolean excludeCompaction) throws IOException {
|
||||||
|
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
|
||||||
HoodieTableMetaClient metaClient =
|
HoodieTableMetaClient metaClient =
|
||||||
new HoodieTableMetaClient(HoodieCLI.tableMetadata.getHadoopConf(), HoodieCLI.tableMetadata.getBasePath(), true);
|
new HoodieTableMetaClient(client.getHadoopConf(), client.getBasePath(), true);
|
||||||
FileSystem fs = HoodieCLI.fs;
|
FileSystem fs = HoodieCLI.fs;
|
||||||
String globPath = String.format("%s/%s/*", HoodieCLI.tableMetadata.getBasePath(), globRegex);
|
String globPath = String.format("%s/%s/*", client.getBasePath(), globRegex);
|
||||||
FileStatus[] statuses = fs.globStatus(new Path(globPath));
|
FileStatus[] statuses = fs.globStatus(new Path(globPath));
|
||||||
Stream<HoodieInstant> instantsStream = null;
|
Stream<HoodieInstant> instantsStream = null;
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import org.apache.hudi.cli.TableHeader;
|
|||||||
import org.apache.hudi.common.model.HoodieLogFile;
|
import org.apache.hudi.common.model.HoodieLogFile;
|
||||||
import org.apache.hudi.common.model.HoodieRecord;
|
import org.apache.hudi.common.model.HoodieRecord;
|
||||||
import org.apache.hudi.common.model.HoodieRecordPayload;
|
import org.apache.hudi.common.model.HoodieRecordPayload;
|
||||||
|
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
||||||
import org.apache.hudi.common.table.log.HoodieLogFormat;
|
import org.apache.hudi.common.table.log.HoodieLogFormat;
|
||||||
import org.apache.hudi.common.table.log.HoodieLogFormat.Reader;
|
import org.apache.hudi.common.table.log.HoodieLogFormat.Reader;
|
||||||
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
|
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
|
||||||
@@ -46,7 +47,6 @@ import org.apache.hadoop.fs.FileSystem;
|
|||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.parquet.avro.AvroSchemaConverter;
|
import org.apache.parquet.avro.AvroSchemaConverter;
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -68,11 +68,6 @@ import scala.Tuple3;
|
|||||||
@Component
|
@Component
|
||||||
public class HoodieLogFileCommand implements CommandMarker {
|
public class HoodieLogFileCommand implements CommandMarker {
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"show logfiles"})
|
|
||||||
public boolean isShowArchivedLogFileAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "show logfile metadata", help = "Read commit metadata from log files")
|
@CliCommand(value = "show logfile metadata", help = "Read commit metadata from log files")
|
||||||
public String showLogFileCommits(
|
public String showLogFileCommits(
|
||||||
@CliOption(key = "logFilePathPattern", mandatory = true,
|
@CliOption(key = "logFilePathPattern", mandatory = true,
|
||||||
@@ -84,7 +79,7 @@ public class HoodieLogFileCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
FileSystem fs = HoodieCLI.tableMetadata.getFs();
|
FileSystem fs = HoodieCLI.getTableMetaClient().getFs();
|
||||||
List<String> logFilePaths = Arrays.stream(fs.globStatus(new Path(logFilePathPattern)))
|
List<String> logFilePaths = Arrays.stream(fs.globStatus(new Path(logFilePathPattern)))
|
||||||
.map(status -> status.getPath().toString()).collect(Collectors.toList());
|
.map(status -> status.getPath().toString()).collect(Collectors.toList());
|
||||||
Map<String, List<Tuple3<HoodieLogBlockType, Tuple2<Map<HeaderMetadataType, String>, Map<HeaderMetadataType, String>>, Integer>>> commitCountAndMetadata =
|
Map<String, List<Tuple3<HoodieLogBlockType, Tuple2<Map<HeaderMetadataType, String>, Map<HeaderMetadataType, String>>, Integer>>> commitCountAndMetadata =
|
||||||
@@ -96,7 +91,7 @@ public class HoodieLogFileCommand implements CommandMarker {
|
|||||||
for (String logFilePath : logFilePaths) {
|
for (String logFilePath : logFilePaths) {
|
||||||
FileStatus[] fsStatus = fs.listStatus(new Path(logFilePath));
|
FileStatus[] fsStatus = fs.listStatus(new Path(logFilePath));
|
||||||
Schema writerSchema = new AvroSchemaConverter()
|
Schema writerSchema = new AvroSchemaConverter()
|
||||||
.convert(SchemaUtil.readSchemaFromLogFile(HoodieCLI.tableMetadata.getFs(), new Path(logFilePath)));
|
.convert(SchemaUtil.readSchemaFromLogFile(fs, new Path(logFilePath)));
|
||||||
Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema);
|
Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema);
|
||||||
|
|
||||||
// read the avro blocks
|
// read the avro blocks
|
||||||
@@ -178,7 +173,8 @@ public class HoodieLogFileCommand implements CommandMarker {
|
|||||||
|
|
||||||
System.out.println("===============> Showing only " + limit + " records <===============");
|
System.out.println("===============> Showing only " + limit + " records <===============");
|
||||||
|
|
||||||
FileSystem fs = HoodieCLI.tableMetadata.getFs();
|
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
|
||||||
|
FileSystem fs = client.getFs();
|
||||||
List<String> logFilePaths = Arrays.stream(fs.globStatus(new Path(logFilePathPattern)))
|
List<String> logFilePaths = Arrays.stream(fs.globStatus(new Path(logFilePathPattern)))
|
||||||
.map(status -> status.getPath().toString()).collect(Collectors.toList());
|
.map(status -> status.getPath().toString()).collect(Collectors.toList());
|
||||||
|
|
||||||
@@ -193,8 +189,8 @@ public class HoodieLogFileCommand implements CommandMarker {
|
|||||||
if (shouldMerge) {
|
if (shouldMerge) {
|
||||||
System.out.println("===========================> MERGING RECORDS <===================");
|
System.out.println("===========================> MERGING RECORDS <===================");
|
||||||
HoodieMergedLogRecordScanner scanner =
|
HoodieMergedLogRecordScanner scanner =
|
||||||
new HoodieMergedLogRecordScanner(fs, HoodieCLI.tableMetadata.getBasePath(), logFilePaths, readerSchema,
|
new HoodieMergedLogRecordScanner(fs, client.getBasePath(), logFilePaths, readerSchema,
|
||||||
HoodieCLI.tableMetadata.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp(),
|
client.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp(),
|
||||||
Long.valueOf(HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES),
|
Long.valueOf(HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES),
|
||||||
Boolean.valueOf(HoodieCompactionConfig.DEFAULT_COMPACTION_LAZY_BLOCK_READ_ENABLED),
|
Boolean.valueOf(HoodieCompactionConfig.DEFAULT_COMPACTION_LAZY_BLOCK_READ_ENABLED),
|
||||||
Boolean.valueOf(HoodieCompactionConfig.DEFAULT_COMPACTION_REVERSE_LOG_READ_ENABLED),
|
Boolean.valueOf(HoodieCompactionConfig.DEFAULT_COMPACTION_REVERSE_LOG_READ_ENABLED),
|
||||||
@@ -210,7 +206,7 @@ public class HoodieLogFileCommand implements CommandMarker {
|
|||||||
} else {
|
} else {
|
||||||
for (String logFile : logFilePaths) {
|
for (String logFile : logFilePaths) {
|
||||||
Schema writerSchema = new AvroSchemaConverter()
|
Schema writerSchema = new AvroSchemaConverter()
|
||||||
.convert(SchemaUtil.readSchemaFromLogFile(HoodieCLI.tableMetadata.getFs(), new Path(logFile)));
|
.convert(SchemaUtil.readSchemaFromLogFile(client.getFs(), new Path(logFile)));
|
||||||
HoodieLogFormat.Reader reader =
|
HoodieLogFormat.Reader reader =
|
||||||
HoodieLogFormat.newReader(fs, new HoodieLogFile(new Path(logFile)), writerSchema);
|
HoodieLogFormat.newReader(fs, new HoodieLogFile(new Path(logFile)), writerSchema);
|
||||||
// read the avro blocks
|
// read the avro blocks
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ import org.apache.hudi.cli.utils.HiveUtil;
|
|||||||
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
||||||
import org.apache.hudi.common.table.HoodieTimeline;
|
import org.apache.hudi.common.table.HoodieTimeline;
|
||||||
import org.apache.hudi.common.table.timeline.HoodieInstant;
|
import org.apache.hudi.common.table.timeline.HoodieInstant;
|
||||||
|
import org.apache.hudi.exception.HoodieException;
|
||||||
|
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -40,11 +40,6 @@ import java.util.stream.Collectors;
|
|||||||
@Component
|
@Component
|
||||||
public class HoodieSyncCommand implements CommandMarker {
|
public class HoodieSyncCommand implements CommandMarker {
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"sync validate"})
|
|
||||||
public boolean isSyncVerificationAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null && HoodieCLI.syncTableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "sync validate", help = "Validate the sync by counting the number of records")
|
@CliCommand(value = "sync validate", help = "Validate the sync by counting the number of records")
|
||||||
public String validateSync(
|
public String validateSync(
|
||||||
@CliOption(key = {"mode"}, unspecifiedDefaultValue = "complete", help = "Check mode") final String mode,
|
@CliOption(key = {"mode"}, unspecifiedDefaultValue = "complete", help = "Check mode") final String mode,
|
||||||
@@ -60,9 +55,12 @@ public class HoodieSyncCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"hivePass"}, mandatory = true, unspecifiedDefaultValue = "",
|
@CliOption(key = {"hivePass"}, mandatory = true, unspecifiedDefaultValue = "",
|
||||||
help = "hive password to connect to") final String hivePass)
|
help = "hive password to connect to") final String hivePass)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
if (HoodieCLI.syncTableMetadata == null) {
|
||||||
|
throw new HoodieException("Sync validate request target table not null.");
|
||||||
|
}
|
||||||
HoodieTableMetaClient target = HoodieCLI.syncTableMetadata;
|
HoodieTableMetaClient target = HoodieCLI.syncTableMetadata;
|
||||||
HoodieTimeline targetTimeline = target.getActiveTimeline().getCommitsTimeline();
|
HoodieTimeline targetTimeline = target.getActiveTimeline().getCommitsTimeline();
|
||||||
HoodieTableMetaClient source = HoodieCLI.tableMetadata;
|
HoodieTableMetaClient source = HoodieCLI.getTableMetaClient();
|
||||||
HoodieTimeline sourceTimeline = source.getActiveTimeline().getCommitsTimeline();
|
HoodieTimeline sourceTimeline = source.getActiveTimeline().getCommitsTimeline();
|
||||||
long sourceCount = 0;
|
long sourceCount = 0;
|
||||||
long targetCount = 0;
|
long targetCount = 0;
|
||||||
|
|||||||
@@ -23,12 +23,12 @@ import org.apache.hudi.cli.HoodiePrintHelper;
|
|||||||
import org.apache.hudi.cli.utils.InputStreamConsumer;
|
import org.apache.hudi.cli.utils.InputStreamConsumer;
|
||||||
import org.apache.hudi.cli.utils.SparkUtil;
|
import org.apache.hudi.cli.utils.SparkUtil;
|
||||||
import org.apache.hudi.common.model.HoodiePartitionMetadata;
|
import org.apache.hudi.common.model.HoodiePartitionMetadata;
|
||||||
|
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
||||||
import org.apache.hudi.common.util.FSUtils;
|
import org.apache.hudi.common.util.FSUtils;
|
||||||
|
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.spark.launcher.SparkLauncher;
|
import org.apache.spark.launcher.SparkLauncher;
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -42,16 +42,6 @@ import java.util.List;
|
|||||||
@Component
|
@Component
|
||||||
public class RepairsCommand implements CommandMarker {
|
public class RepairsCommand implements CommandMarker {
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"repair deduplicate"})
|
|
||||||
public boolean isRepairDeduplicateAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"repair addpartitionmeta"})
|
|
||||||
public boolean isRepairAddPartitionMetaAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "repair deduplicate",
|
@CliCommand(value = "repair deduplicate",
|
||||||
help = "De-duplicate a partition path contains duplicates & produce repaired files to replace with")
|
help = "De-duplicate a partition path contains duplicates & produce repaired files to replace with")
|
||||||
public String deduplicate(
|
public String deduplicate(
|
||||||
@@ -64,7 +54,7 @@ public class RepairsCommand implements CommandMarker {
|
|||||||
throws Exception {
|
throws Exception {
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkMain.SparkCommand.DEDUPLICATE.toString(), duplicatedPartitionPath, repairedOutputPath,
|
sparkLauncher.addAppArgs(SparkMain.SparkCommand.DEDUPLICATE.toString(), duplicatedPartitionPath, repairedOutputPath,
|
||||||
HoodieCLI.tableMetadata.getBasePath());
|
HoodieCLI.getTableMetaClient().getBasePath());
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
InputStreamConsumer.captureOutput(process);
|
InputStreamConsumer.captureOutput(process);
|
||||||
int exitCode = process.waitFor();
|
int exitCode = process.waitFor();
|
||||||
@@ -81,11 +71,12 @@ public class RepairsCommand implements CommandMarker {
|
|||||||
unspecifiedDefaultValue = "true") final boolean dryRun)
|
unspecifiedDefaultValue = "true") final boolean dryRun)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
|
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
|
||||||
String latestCommit =
|
String latestCommit =
|
||||||
HoodieCLI.tableMetadata.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp();
|
client.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp();
|
||||||
List<String> partitionPaths =
|
List<String> partitionPaths =
|
||||||
FSUtils.getAllPartitionFoldersThreeLevelsDown(HoodieCLI.fs, HoodieCLI.tableMetadata.getBasePath());
|
FSUtils.getAllPartitionFoldersThreeLevelsDown(HoodieCLI.fs, client.getBasePath());
|
||||||
Path basePath = new Path(HoodieCLI.tableMetadata.getBasePath());
|
Path basePath = new Path(client.getBasePath());
|
||||||
String[][] rows = new String[partitionPaths.size() + 1][];
|
String[][] rows = new String[partitionPaths.size() + 1][];
|
||||||
|
|
||||||
int ind = 0;
|
int ind = 0;
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ public class RollbacksCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
HoodieActiveTimeline activeTimeline = new RollbackTimeline(HoodieCLI.tableMetadata);
|
HoodieActiveTimeline activeTimeline = new RollbackTimeline(HoodieCLI.getTableMetaClient());
|
||||||
HoodieTimeline rollback = activeTimeline.getRollbackTimeline().filterCompletedInstants();
|
HoodieTimeline rollback = activeTimeline.getRollbackTimeline().filterCompletedInstants();
|
||||||
|
|
||||||
final List<Comparable[]> rows = new ArrayList<>();
|
final List<Comparable[]> rows = new ArrayList<>();
|
||||||
@@ -94,7 +94,7 @@ public class RollbacksCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
@CliOption(key = {"headeronly"}, help = "Print Header Only",
|
||||||
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
unspecifiedDefaultValue = "false") final boolean headerOnly)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
HoodieActiveTimeline activeTimeline = new RollbackTimeline(HoodieCLI.tableMetadata);
|
HoodieActiveTimeline activeTimeline = new RollbackTimeline(HoodieCLI.getTableMetaClient());
|
||||||
final List<Comparable[]> rows = new ArrayList<>();
|
final List<Comparable[]> rows = new ArrayList<>();
|
||||||
HoodieRollbackMetadata metadata = AvroUtils.deserializeAvroMetadata(
|
HoodieRollbackMetadata metadata = AvroUtils.deserializeAvroMetadata(
|
||||||
activeTimeline.getInstantDetails(new HoodieInstant(State.COMPLETED, ROLLBACK_ACTION, rollbackInstant)).get(),
|
activeTimeline.getInstantDetails(new HoodieInstant(State.COMPLETED, ROLLBACK_ACTION, rollbackInstant)).get(),
|
||||||
|
|||||||
@@ -23,17 +23,18 @@ import org.apache.hudi.cli.HoodieCLI;
|
|||||||
import org.apache.hudi.cli.HoodiePrintHelper;
|
import org.apache.hudi.cli.HoodiePrintHelper;
|
||||||
import org.apache.hudi.cli.utils.InputStreamConsumer;
|
import org.apache.hudi.cli.utils.InputStreamConsumer;
|
||||||
import org.apache.hudi.cli.utils.SparkUtil;
|
import org.apache.hudi.cli.utils.SparkUtil;
|
||||||
|
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
||||||
import org.apache.hudi.common.table.HoodieTimeline;
|
import org.apache.hudi.common.table.HoodieTimeline;
|
||||||
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
|
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
|
||||||
import org.apache.hudi.common.table.timeline.HoodieInstant;
|
import org.apache.hudi.common.table.timeline.HoodieInstant;
|
||||||
import org.apache.hudi.config.HoodieIndexConfig;
|
import org.apache.hudi.config.HoodieIndexConfig;
|
||||||
import org.apache.hudi.config.HoodieWriteConfig;
|
import org.apache.hudi.config.HoodieWriteConfig;
|
||||||
|
import org.apache.hudi.exception.HoodieException;
|
||||||
import org.apache.hudi.index.HoodieIndex;
|
import org.apache.hudi.index.HoodieIndex;
|
||||||
|
|
||||||
import org.apache.spark.api.java.JavaSparkContext;
|
import org.apache.spark.api.java.JavaSparkContext;
|
||||||
import org.apache.spark.launcher.SparkLauncher;
|
import org.apache.spark.launcher.SparkLauncher;
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -48,30 +49,9 @@ import java.util.stream.Collectors;
|
|||||||
@Component
|
@Component
|
||||||
public class SavepointsCommand implements CommandMarker {
|
public class SavepointsCommand implements CommandMarker {
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"savepoints show"})
|
|
||||||
public boolean isShowAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"savepoints refresh"})
|
|
||||||
public boolean isRefreshAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"savepoint create"})
|
|
||||||
public boolean isCreateSavepointAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"savepoint rollback"})
|
|
||||||
public boolean isRollbackToSavepointAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null
|
|
||||||
&& !HoodieCLI.tableMetadata.getActiveTimeline().getSavePointTimeline().filterCompletedInstants().empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "savepoints show", help = "Show the savepoints")
|
@CliCommand(value = "savepoints show", help = "Show the savepoints")
|
||||||
public String showSavepoints() throws IOException {
|
public String showSavepoints() throws IOException {
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getSavePointTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getSavePointTimeline().filterCompletedInstants();
|
||||||
List<HoodieInstant> commits = timeline.getReverseOrderedInstants().collect(Collectors.toList());
|
List<HoodieInstant> commits = timeline.getReverseOrderedInstants().collect(Collectors.toList());
|
||||||
String[][] rows = new String[commits.size()][];
|
String[][] rows = new String[commits.size()][];
|
||||||
@@ -87,7 +67,8 @@ public class SavepointsCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"user"}, unspecifiedDefaultValue = "default", help = "User who is creating the savepoint") final String user,
|
@CliOption(key = {"user"}, unspecifiedDefaultValue = "default", help = "User who is creating the savepoint") final String user,
|
||||||
@CliOption(key = {"comments"}, unspecifiedDefaultValue = "default", help = "Comments for creating the savepoint") final String comments)
|
@CliOption(key = {"comments"}, unspecifiedDefaultValue = "default", help = "Comments for creating the savepoint") final String comments)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
|
||||||
|
HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
||||||
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
||||||
|
|
||||||
@@ -96,7 +77,7 @@ public class SavepointsCommand implements CommandMarker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
JavaSparkContext jsc = SparkUtil.initJavaSparkConf("Create Savepoint");
|
JavaSparkContext jsc = SparkUtil.initJavaSparkConf("Create Savepoint");
|
||||||
HoodieWriteClient client = createHoodieClient(jsc, HoodieCLI.tableMetadata.getBasePath());
|
HoodieWriteClient client = createHoodieClient(jsc, metaClient.getBasePath());
|
||||||
String result;
|
String result;
|
||||||
if (client.savepoint(commitTime, user, comments)) {
|
if (client.savepoint(commitTime, user, comments)) {
|
||||||
// Refresh the current
|
// Refresh the current
|
||||||
@@ -114,7 +95,11 @@ public class SavepointsCommand implements CommandMarker {
|
|||||||
@CliOption(key = {"savepoint"}, help = "Savepoint to rollback") final String commitTime,
|
@CliOption(key = {"savepoint"}, help = "Savepoint to rollback") final String commitTime,
|
||||||
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path") final String sparkPropertiesPath)
|
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path") final String sparkPropertiesPath)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
|
||||||
|
if (metaClient.getActiveTimeline().getSavePointTimeline().filterCompletedInstants().empty()) {
|
||||||
|
throw new HoodieException("There are no completed instants to run rollback");
|
||||||
|
}
|
||||||
|
HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
||||||
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
|
||||||
|
|
||||||
@@ -124,7 +109,7 @@ public class SavepointsCommand implements CommandMarker {
|
|||||||
|
|
||||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||||
sparkLauncher.addAppArgs(SparkMain.SparkCommand.ROLLBACK_TO_SAVEPOINT.toString(), commitTime,
|
sparkLauncher.addAppArgs(SparkMain.SparkCommand.ROLLBACK_TO_SAVEPOINT.toString(), commitTime,
|
||||||
HoodieCLI.tableMetadata.getBasePath());
|
metaClient.getBasePath());
|
||||||
Process process = sparkLauncher.launch();
|
Process process = sparkLauncher.launch();
|
||||||
InputStreamConsumer.captureOutput(process);
|
InputStreamConsumer.captureOutput(process);
|
||||||
int exitCode = process.waitFor();
|
int exitCode = process.waitFor();
|
||||||
@@ -139,7 +124,7 @@ public class SavepointsCommand implements CommandMarker {
|
|||||||
@CliCommand(value = "savepoints refresh", help = "Refresh the savepoints")
|
@CliCommand(value = "savepoints refresh", help = "Refresh the savepoints")
|
||||||
public String refreshMetaClient() throws IOException {
|
public String refreshMetaClient() throws IOException {
|
||||||
HoodieCLI.refreshTableMetadata();
|
HoodieCLI.refreshTableMetadata();
|
||||||
return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " refreshed.";
|
return "Metadata for table " + HoodieCLI.getTableMetaClient().getTableConfig().getTableName() + " refreshed.";
|
||||||
}
|
}
|
||||||
|
|
||||||
private static HoodieWriteClient createHoodieClient(JavaSparkContext jsc, String basePath) throws Exception {
|
private static HoodieWriteClient createHoodieClient(JavaSparkContext jsc, String basePath) throws Exception {
|
||||||
|
|||||||
@@ -35,7 +35,6 @@ import org.apache.hadoop.fs.FileStatus;
|
|||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.springframework.shell.core.CommandMarker;
|
import org.springframework.shell.core.CommandMarker;
|
||||||
import org.springframework.shell.core.annotation.CliAvailabilityIndicator;
|
|
||||||
import org.springframework.shell.core.annotation.CliCommand;
|
import org.springframework.shell.core.annotation.CliCommand;
|
||||||
import org.springframework.shell.core.annotation.CliOption;
|
import org.springframework.shell.core.annotation.CliOption;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@@ -57,11 +56,6 @@ public class StatsCommand implements CommandMarker {
|
|||||||
|
|
||||||
private static final int MAX_FILES = 1000000;
|
private static final int MAX_FILES = 1000000;
|
||||||
|
|
||||||
@CliAvailabilityIndicator({"stats wa"})
|
|
||||||
public boolean isWriteAmpAvailable() {
|
|
||||||
return HoodieCLI.tableMetadata != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliCommand(value = "stats wa", help = "Write Amplification. Ratio of how many records were upserted to how many "
|
@CliCommand(value = "stats wa", help = "Write Amplification. Ratio of how many records were upserted to how many "
|
||||||
+ "records were actually written")
|
+ "records were actually written")
|
||||||
public String writeAmplificationStats(
|
public String writeAmplificationStats(
|
||||||
@@ -75,7 +69,7 @@ public class StatsCommand implements CommandMarker {
|
|||||||
long totalRecordsUpserted = 0;
|
long totalRecordsUpserted = 0;
|
||||||
long totalRecordsWritten = 0;
|
long totalRecordsWritten = 0;
|
||||||
|
|
||||||
HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
|
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
|
||||||
HoodieTimeline timeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
HoodieTimeline timeline = activeTimeline.getCommitTimeline().filterCompletedInstants();
|
||||||
|
|
||||||
List<Comparable[]> rows = new ArrayList<>();
|
List<Comparable[]> rows = new ArrayList<>();
|
||||||
@@ -121,7 +115,7 @@ public class StatsCommand implements CommandMarker {
|
|||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
FileSystem fs = HoodieCLI.fs;
|
FileSystem fs = HoodieCLI.fs;
|
||||||
String globPath = String.format("%s/%s/*", HoodieCLI.tableMetadata.getBasePath(), globRegex);
|
String globPath = String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex);
|
||||||
FileStatus[] statuses = fs.globStatus(new Path(globPath));
|
FileStatus[] statuses = fs.globStatus(new Path(globPath));
|
||||||
|
|
||||||
// max, min, #small files < 10MB, 50th, avg, 95th
|
// max, min, #small files < 10MB, 50th, avg, 95th
|
||||||
|
|||||||
@@ -77,14 +77,13 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
|
|||||||
* Ensures each instant time is atleast 1 second apart since we create instant times at second granularity
|
* Ensures each instant time is atleast 1 second apart since we create instant times at second granularity
|
||||||
*/
|
*/
|
||||||
public static String createNewInstantTime() {
|
public static String createNewInstantTime() {
|
||||||
lastInstantTime.updateAndGet((oldVal) -> {
|
return lastInstantTime.updateAndGet((oldVal) -> {
|
||||||
String newCommitTime = null;
|
String newCommitTime = null;
|
||||||
do {
|
do {
|
||||||
newCommitTime = HoodieActiveTimeline.COMMIT_FORMATTER.format(new Date());
|
newCommitTime = HoodieActiveTimeline.COMMIT_FORMATTER.format(new Date());
|
||||||
} while (HoodieTimeline.compareTimestamps(newCommitTime, oldVal, LESSER_OR_EQUAL));
|
} while (HoodieTimeline.compareTimestamps(newCommitTime, oldVal, LESSER_OR_EQUAL));
|
||||||
return newCommitTime;
|
return newCommitTime;
|
||||||
});
|
});
|
||||||
return lastInstantTime.get();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected HoodieActiveTimeline(HoodieTableMetaClient metaClient, Set<String> includedExtensions) {
|
protected HoodieActiveTimeline(HoodieTableMetaClient metaClient, Set<String> includedExtensions) {
|
||||||
|
|||||||
Reference in New Issue
Block a user