lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1486] Remove inline inflight rollback in hoodie writer (#2359)

Browse Source 
1. Refactor rollback and move cleaning failed commits logic into cleaner
2. Introduce hoodie heartbeat to ascertain failed commits
3. Fix test cases
This commit is contained in:
n3nash
2021-02-19 20:12:22 -08:00
committed by GitHub
parent c9fcf964b2
commit ffcfb58bac
64 changed files with 1541 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.java
View File

@@ -78,13 +78,7 @@ public class HoodieTestSuiteWriter {
Arrays.asList(RollbackNode.class.getName(), CleanNode.class.getName(), ScheduleCompactNode.class.getName()));
private static final String GENERATED_DATA_PATH = "generated.data.path";
public HoodieTestSuiteWriter(JavaSparkContext jsc, Properties props, HoodieTestSuiteConfig cfg, String schema) throws
Exception {
this(jsc, props, cfg, schema, true);
}
public HoodieTestSuiteWriter(JavaSparkContext jsc, Properties props, HoodieTestSuiteConfig cfg, String schema,
boolean rollbackInflight) throws Exception {
public HoodieTestSuiteWriter(JavaSparkContext jsc, Properties props, HoodieTestSuiteConfig cfg, String schema) throws Exception {
// We ensure that only 1 instance of HoodieWriteClient is instantiated for a HoodieTestSuiteWriter
// This does not instantiate a HoodieWriteClient until a
// {@link HoodieDeltaStreamer#commit(HoodieWriteClient, JavaRDD, Option)} is invoked.
@@ -93,7 +87,7 @@ public class HoodieTestSuiteWriter {
this.hoodieReadClient = new HoodieReadClient(context, cfg.targetBasePath);
this.writeConfig = getHoodieClientConfig(cfg, props, schema);
if (!cfg.useDeltaStreamer) {
this.writeClient = new SparkRDDWriteClient(context, writeConfig, rollbackInflight);
this.writeClient = new SparkRDDWriteClient(context, writeConfig);
}
this.cfg = cfg;
this.configuration = jsc.hadoopConfiguration();
@@ -234,7 +228,7 @@ public class HoodieTestSuiteWriter {
}
synchronized (this) {
if (writeClient == null) {
this.writeClient = new SparkRDDWriteClient(new HoodieSparkEngineContext(this.sparkContext), getHoodieClientConfig(cfg, props, schema), false);
this.writeClient = new SparkRDDWriteClient(new HoodieSparkEngineContext(this.sparkContext), getHoodieClientConfig(cfg, props, schema));
}
}
return writeClient;

2
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CompactNode.java
View File

@@ -49,7 +49,7 @@ public class CompactNode extends DagNode<JavaRDD<WriteStatus>> {
HoodieTableMetaClient.builder().setConf(executionContext.getHoodieTestSuiteWriter().getConfiguration()).setBasePath(executionContext.getHoodieTestSuiteWriter().getCfg().targetBasePath)
.build();
Option<HoodieInstant> lastInstant = metaClient.getActiveTimeline()
.getCommitsAndCompactionTimeline().filterPendingCompactionTimeline().lastInstant();
.getWriteTimeline().filterPendingCompactionTimeline().lastInstant();
if (lastInstant.isPresent()) {
log.info("Compacting instant {}", lastInstant.get());
this.result = executionContext.getHoodieTestSuiteWriter().compact(Option.of(lastInstant.get().getTimestamp()));