lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2530] Adding async compaction support to integ test suite framework (#3750)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-10-08 11:30:48 -04:00
committed by GitHub
parent 10e3a9a3fb
commit a818020f72
6 changed files with 198 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.java
View File

@@ -18,6 +18,7 @@
package org.apache.hudi.integ.testsuite;
import java.io.IOException;
import java.io.Serializable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
@@ -236,7 +237,7 @@ public class HoodieTestSuiteWriter implements Serializable {
public Option<String> scheduleCompaction(Option<Map<String, String>> previousCommitExtraMetadata) throws
Exception {
if (!cfg.useDeltaStreamer) {
if (cfg.useDeltaStreamer) {
deltaStreamerWrapper.scheduleCompact();
return Option.empty();
} else {
@@ -251,7 +252,7 @@ public class HoodieTestSuiteWriter implements Serializable {
/** Store the checkpoint in the commit metadata just like
* {@link HoodieDeltaStreamer#commit(SparkRDDWriteClient, JavaRDD, Option)} **/
extraMetadata.put(HoodieDeltaStreamerWrapper.CHECKPOINT_KEY, lastCheckpoint.get());
if (generatedDataStats != null) {
if (generatedDataStats != null && generatedDataStats.count() > 1) {
// Just stores the path where this batch of data is generated to
extraMetadata.put(GENERATED_DATA_PATH, generatedDataStats.map(s -> s.getFilePath()).collect().get(0));
}
@@ -259,6 +260,21 @@ public class HoodieTestSuiteWriter implements Serializable {
}
}
public void commitCompaction(JavaRDD<WriteStatus> records, JavaRDD<DeltaWriteStats> generatedDataStats,
Option<String> instantTime) throws IOException {
if (!cfg.useDeltaStreamer) {
Map<String, String> extraMetadata = new HashMap<>();
/** Store the checkpoint in the commit metadata just like
* {@link HoodieDeltaStreamer#commit(SparkRDDWriteClient, JavaRDD, Option)} **/
extraMetadata.put(HoodieDeltaStreamerWrapper.CHECKPOINT_KEY, lastCheckpoint.get());
if (generatedDataStats != null && generatedDataStats.count() > 1) {
// Just stores the path where this batch of data is generated to
extraMetadata.put(GENERATED_DATA_PATH, generatedDataStats.map(s -> s.getFilePath()).collect().get(0));
}
writeClient.commitCompaction(instantTime.get(), records, Option.of(extraMetadata));
}
}
public SparkRDDWriteClient getWriteClient(DagNode dagNode) throws IllegalAccessException {
if (cfg.useDeltaStreamer & !allowWriteClientAccess(dagNode)) {
throw new IllegalAccessException("cannot access write client when testing in deltastreamer mode");

7
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CompactNode.java
View File

@@ -24,6 +24,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
import org.apache.spark.api.java.JavaRDD;
/**
@@ -40,8 +41,8 @@ public class CompactNode extends DagNode<JavaRDD<WriteStatus>> {
* if it has one.
*
* @param executionContext Execution context to run this compaction
* @param curItrCount cur interation count.
* @throws Exception will be thrown if any error occurred.
* @param curItrCount cur interation count.
* @throws Exception will be thrown if any error occurred.
*/
@Override
public void execute(ExecutionContext executionContext, int curItrCount) throws Exception {
@@ -53,7 +54,7 @@ public class CompactNode extends DagNode<JavaRDD<WriteStatus>> {
if (lastInstant.isPresent()) {
log.info("Compacting instant {}", lastInstant.get());
this.result = executionContext.getHoodieTestSuiteWriter().compact(Option.of(lastInstant.get().getTimestamp()));
executionContext.getHoodieTestSuiteWriter().commitCompaction(result, executionContext.getJsc().emptyRDD(), Option.of(lastInstant.get().getTimestamp()));
}
}
}

1
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ScheduleCompactNode.java
View File

@@ -56,5 +56,4 @@ public class ScheduleCompactNode extends DagNode<Option<String>> {
this.result = scheduledInstant;
}
}
}