lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2747] support set --sparkMaster for MDT cli (#4964)

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
YueZhang
2022-03-08 05:57:03 +08:00
committed by GitHub
parent a66fd40692
commit 53826d69e4
1 changed files with 13 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
View File

@@ -108,7 +108,9 @@ public class MetadataCommand implements CommandMarker {
} }
@CliCommand(value = "metadata create", help = "Create the Metadata Table if it does not exist") @CliCommand(value = "metadata create", help = "Create the Metadata Table if it does not exist")
public String create() throws IOException { public String create(
@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
) throws IOException {
HoodieCLI.getTableMetaClient(); HoodieCLI.getTableMetaClient();
Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath)); Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
try { try {
@@ -123,7 +125,7 @@ public class MetadataCommand implements CommandMarker {
HoodieTimer timer = new HoodieTimer().startTimer(); HoodieTimer timer = new HoodieTimer().startTimer();
HoodieWriteConfig writeConfig = getWriteConfig(); HoodieWriteConfig writeConfig = getWriteConfig();
initJavaSparkContext(); initJavaSparkContext(Option.of(master));
SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc)); SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
return String.format("Created Metadata Table in %s (duration=%.2f secs)", metadataPath, timer.endTimer() / 1000.0); return String.format("Created Metadata Table in %s (duration=%.2f secs)", metadataPath, timer.endTimer() / 1000.0);
} }
@@ -145,7 +147,8 @@ public class MetadataCommand implements CommandMarker {
} }
@CliCommand(value = "metadata init", help = "Update the metadata table from commits since the creation") @CliCommand(value = "metadata init", help = "Update the metadata table from commits since the creation")
public String init(@CliOption(key = {"readonly"}, unspecifiedDefaultValue = "false", public String init(@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master,
@CliOption(key = {"readonly"}, unspecifiedDefaultValue = "false",
help = "Open in read-only mode") final boolean readOnly) throws Exception { help = "Open in read-only mode") final boolean readOnly) throws Exception {
HoodieCLI.getTableMetaClient(); HoodieCLI.getTableMetaClient();
Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath)); Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
@@ -159,7 +162,7 @@ public class MetadataCommand implements CommandMarker {
HoodieTimer timer = new HoodieTimer().startTimer(); HoodieTimer timer = new HoodieTimer().startTimer();
if (!readOnly) { if (!readOnly) {
HoodieWriteConfig writeConfig = getWriteConfig(); HoodieWriteConfig writeConfig = getWriteConfig();
initJavaSparkContext(); initJavaSparkContext(Option.of(master));
SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc)); SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
} }
@@ -191,9 +194,11 @@ public class MetadataCommand implements CommandMarker {
} }
@CliCommand(value = "metadata list-partitions", help = "List all partitions from metadata") @CliCommand(value = "metadata list-partitions", help = "List all partitions from metadata")
public String listPartitions() throws IOException { public String listPartitions(
@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
) throws IOException {
HoodieCLI.getTableMetaClient(); HoodieCLI.getTableMetaClient();
initJavaSparkContext(); initJavaSparkContext(Option.of(master));
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build(); HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config, HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config,
HoodieCLI.basePath, "/tmp"); HoodieCLI.basePath, "/tmp");
@@ -357,9 +362,9 @@ public class MetadataCommand implements CommandMarker {
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build(); .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build();
} }
private void initJavaSparkContext() { private void initJavaSparkContext(Option<String> userDefinedMaster) {
if (jsc == null) { if (jsc == null) {
jsc = SparkUtil.initJavaSparkConf(SparkUtil.getDefaultConf("HoodieCLI", Option.empty())); jsc = SparkUtil.initJavaSparkConf(SparkUtil.getDefaultConf("HoodieCLI", userDefinedMaster));
} }
} }
} }