lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3369] New ScheduleAndExecute mode for HoodieCompactor and hudi-cli (#4750)

Browse Source 
Schedule and execute compaction plan in one single mode.
This commit is contained in:
YueZhang
2022-02-07 17:31:34 +08:00
committed by GitHub
parent 0880a8a5e4
commit de206acbae
4 changed files with 249 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
View File

@@ -18,13 +18,14 @@
package org.apache.hudi.utilities;
import org.apache.avro.Schema;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
@@ -35,6 +36,9 @@ import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.table.action.compact.strategy.LogFileSizeBasedCompactionStrategy;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
@@ -43,15 +47,19 @@ import org.apache.spark.api.java.JavaSparkContext;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class HoodieCompactor {
private static final Logger LOG = LogManager.getLogger(HoodieCompactor.class);
private static ConsistencyGuardConfig consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().build();
public static final String EXECUTE = "execute";
public static final String SCHEDULE = "schedule";
public static final String SCHEDULE_AND_EXECUTE = "scheduleandexecute";
private final Config cfg;
private transient FileSystem fs;
private TypedProperties props;
private final JavaSparkContext jsc;
private final HoodieTableMetaClient metaClient;
public HoodieCompactor(JavaSparkContext jsc, Config cfg) {
this.cfg = cfg;
@@ -59,6 +67,7 @@ public class HoodieCompactor {
this.props = cfg.propsFilePath == null
? UtilHelpers.buildProperties(cfg.configs)
: readConfigFromFileSystem(jsc, cfg);
this.metaClient = UtilHelpers.createMetaClient(jsc, cfg.basePath, true);
}
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
@@ -73,9 +82,9 @@ public class HoodieCompactor {
public String tableName = null;
@Parameter(names = {"--instant-time", "-it"}, description = "Compaction Instant time", required = false)
public String compactionInstantTime = null;
@Parameter(names = {"--parallelism", "-pl"}, description = "Parallelism for hoodie insert", required = true)
public int parallelism = 1;
@Parameter(names = {"--schema-file", "-sf"}, description = "path for Avro schema file", required = true)
@Parameter(names = {"--parallelism", "-pl"}, description = "Parallelism for hoodie insert", required = false)
public int parallelism = 200;
@Parameter(names = {"--schema-file", "-sf"}, description = "path for Avro schema file", required = false)
public String schemaFile = null;
@Parameter(names = {"--spark-master", "-ms"}, description = "Spark master", required = false)
public String sparkMaster = null;
@@ -85,8 +94,12 @@ public class HoodieCompactor {
public int retry = 0;
@Parameter(names = {"--schedule", "-sc"}, description = "Schedule compaction", required = false)
public Boolean runSchedule = false;
@Parameter(names = {"--mode", "-m"}, description = "Set job mode: Set \"schedule\" means make a compact plan; "
+ "Set \"execute\" means execute a compact plan at given instant which means --instant-time is needed here; "
+ "Set \"scheduleAndExecute\" means make a compact plan first and execute that plan immediately", required = false)
public String runningMode = null;
@Parameter(names = {"--strategy", "-st"}, description = "Strategy Class", required = false)
public String strategyClassName = null;
public String strategyClassName = LogFileSizeBasedCompactionStrategy.class.getName();
@Parameter(names = {"--help", "-h"}, help = true)
public Boolean help = false;
@@ -96,8 +109,57 @@ public class HoodieCompactor {
@Parameter(names = {"--hoodie-conf"}, description = "Any configuration that can be set in the properties file "
+ "(using the CLI parameter \"--props\") can also be passed command line using this parameter. This can be repeated",
splitter = IdentitySplitter.class)
splitter = IdentitySplitter.class)
public List<String> configs = new ArrayList<>();
@Override
public String toString() {
return "HoodieCompactorConfig {\n"
+ " --base-path " + basePath + ", \n"
+ " --table-name " + tableName + ", \n"
+ " --instant-time " + compactionInstantTime + ", \n"
+ " --parallelism " + parallelism + ", \n"
+ " --schema-file " + schemaFile + ", \n"
+ " --spark-master " + sparkMaster + ", \n"
+ " --spark-memory " + sparkMemory + ", \n"
+ " --retry " + retry + ", \n"
+ " --schedule " + runSchedule + ", \n"
+ " --mode " + runningMode + ", \n"
+ " --strategy " + strategyClassName + ", \n"
+ " --props " + propsFilePath + ", \n"
+ " --hoodie-conf " + configs
+ "\n}";
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Config config = (Config) o;
return basePath.equals(config.basePath)
&& Objects.equals(tableName, config.tableName)
&& Objects.equals(compactionInstantTime, config.compactionInstantTime)
&& Objects.equals(parallelism, config.parallelism)
&& Objects.equals(schemaFile, config.schemaFile)
&& Objects.equals(sparkMaster, config.sparkMaster)
&& Objects.equals(sparkMemory, config.sparkMemory)
&& Objects.equals(retry, config.retry)
&& Objects.equals(runSchedule, config.runSchedule)
&& Objects.equals(runningMode, config.runningMode)
&& Objects.equals(strategyClassName, config.strategyClassName)
&& Objects.equals(propsFilePath, config.propsFilePath)
&& Objects.equals(configs, config.configs);
}
@Override
public int hashCode() {
return Objects.hash(basePath, tableName, compactionInstantTime, schemaFile,
sparkMaster, parallelism, sparkMemory, retry, runSchedule, runningMode, strategyClassName, propsFilePath, configs, help);
}
}
public static void main(String[] args) {
@@ -120,52 +182,115 @@ public class HoodieCompactor {
public int compact(int retry) {
this.fs = FSUtils.getFs(cfg.basePath, jsc.hadoopConfiguration());
// need to do validate in case that users call compact() directly without setting cfg.runningMode
validateRunningMode(cfg);
LOG.info(cfg);
int ret = UtilHelpers.retry(retry, () -> {
if (cfg.runSchedule) {
if (null == cfg.strategyClassName) {
throw new IllegalArgumentException("Missing Strategy class name for running compaction");
switch (cfg.runningMode.toLowerCase()) {
case SCHEDULE: {
LOG.info("Running Mode: [" + SCHEDULE + "]; Do schedule");
Option<String> instantTime = doSchedule(jsc);
int result = instantTime.isPresent() ? 0 : -1;
if (result == 0) {
LOG.info("The schedule instant time is " + instantTime.get());
}
return result;
}
case SCHEDULE_AND_EXECUTE: {
LOG.info("Running Mode: [" + SCHEDULE_AND_EXECUTE + "]");
return doScheduleAndCompact(jsc);
}
case EXECUTE: {
LOG.info("Running Mode: [" + EXECUTE + "]; Do compaction");
return doCompact(jsc);
}
default: {
LOG.info("Unsupported running mode [" + cfg.runningMode + "], quit the job directly");
return -1;
}
return doSchedule(jsc);
} else {
return doCompact(jsc);
}
}, "Compact failed");
return ret;
}
private int doCompact(JavaSparkContext jsc) throws Exception {
// Get schema.
String schemaStr = UtilHelpers.parseSchema(fs, cfg.schemaFile);
SparkRDDWriteClient<HoodieRecordPayload> client =
UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props);
// If no compaction instant is provided by --instant-time, find the earliest scheduled compaction
// instant from the active timeline
if (StringUtils.isNullOrEmpty(cfg.compactionInstantTime)) {
HoodieTableMetaClient metaClient = UtilHelpers.createMetaClient(jsc, cfg.basePath, true);
Option<HoodieInstant> firstCompactionInstant =
metaClient.getActiveTimeline().firstInstant(
HoodieTimeline.COMPACTION_ACTION, HoodieInstant.State.REQUESTED);
if (firstCompactionInstant.isPresent()) {
cfg.compactionInstantTime = firstCompactionInstant.get().getTimestamp();
LOG.info("Found the earliest scheduled compaction instant which will be executed: "
+ cfg.compactionInstantTime);
} else {
throw new HoodieCompactionException("There is no scheduled compaction in the table.");
}
private Integer doScheduleAndCompact(JavaSparkContext jsc) throws Exception {
LOG.info("Step 1: Do schedule");
Option<String> instantTime = doSchedule(jsc);
if (!instantTime.isPresent()) {
LOG.warn("Couldn't do schedule");
return -1;
} else {
cfg.compactionInstantTime = instantTime.get();
}
JavaRDD<WriteStatus> writeResponse = client.compact(cfg.compactionInstantTime);
return UtilHelpers.handleErrors(jsc, cfg.compactionInstantTime, writeResponse);
LOG.info("The schedule instant time is " + instantTime.get());
LOG.info("Step 2: Do compaction");
return doCompact(jsc);
}
private int doSchedule(JavaSparkContext jsc) throws Exception {
// Get schema.
SparkRDDWriteClient client =
UtilHelpers.createHoodieClient(jsc, cfg.basePath, "", cfg.parallelism, Option.of(cfg.strategyClassName), props);
if (StringUtils.isNullOrEmpty(cfg.compactionInstantTime)) {
throw new IllegalArgumentException("No instant time is provided for scheduling compaction. "
+ "Please specify the compaction instant time by using --instant-time.");
// make sure that cfg.runningMode couldn't be null
private static void validateRunningMode(Config cfg) {
// --mode has a higher priority than --schedule
// If we remove --schedule option in the future we need to change runningMode default value to EXECUTE
if (StringUtils.isNullOrEmpty(cfg.runningMode)) {
cfg.runningMode = cfg.runSchedule ? SCHEDULE : EXECUTE;
}
client.scheduleCompactionAtInstant(cfg.compactionInstantTime, Option.empty());
return 0;
}
private int doCompact(JavaSparkContext jsc) throws Exception {
// Get schema.
String schemaStr;
if (StringUtils.isNullOrEmpty(cfg.schemaFile)) {
schemaStr = getSchemaFromLatestInstant();
} else {
schemaStr = UtilHelpers.parseSchema(fs, cfg.schemaFile);
}
LOG.info("Schema --> : " + schemaStr);
try (SparkRDDWriteClient<HoodieRecordPayload> client =
UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) {
// If no compaction instant is provided by --instant-time, find the earliest scheduled compaction
// instant from the active timeline
if (StringUtils.isNullOrEmpty(cfg.compactionInstantTime)) {
HoodieTableMetaClient metaClient = UtilHelpers.createMetaClient(jsc, cfg.basePath, true);
Option<HoodieInstant> firstCompactionInstant =
metaClient.getActiveTimeline().firstInstant(
HoodieTimeline.COMPACTION_ACTION, HoodieInstant.State.REQUESTED);
if (firstCompactionInstant.isPresent()) {
cfg.compactionInstantTime = firstCompactionInstant.get().getTimestamp();
LOG.info("Found the earliest scheduled compaction instant which will be executed: "
+ cfg.compactionInstantTime);
} else {
throw new HoodieCompactionException("There is no scheduled compaction in the table.");
}
}
JavaRDD<WriteStatus> writeResponse = client.compact(cfg.compactionInstantTime);
return UtilHelpers.handleErrors(jsc, cfg.compactionInstantTime, writeResponse);
}
}
private Option<String> doSchedule(JavaSparkContext jsc) {
try (SparkRDDWriteClient client =
UtilHelpers.createHoodieClient(jsc, cfg.basePath, "", cfg.parallelism, Option.of(cfg.strategyClassName), props)) {
if (StringUtils.isNullOrEmpty(cfg.compactionInstantTime)) {
LOG.warn("No instant time is provided for scheduling compaction.");
return client.scheduleCompaction(Option.empty());
}
client.scheduleCompactionAtInstant(cfg.compactionInstantTime, Option.empty());
return Option.of(cfg.compactionInstantTime);
}
}
private String getSchemaFromLatestInstant() throws Exception {
TableSchemaResolver schemaUtil = new TableSchemaResolver(metaClient);
if (metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().countInstants() == 0) {
throw new HoodieException("Cannot run compaction without any completed commits");
}
Schema schema = schemaUtil.getTableAvroSchema(false);
return schema.toString();
}
}