lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-603]: DeltaStreamer can now fetch schema before every run in continuous mode (#1566)

Browse Source 
Co-authored-by: Balaji Varadarajan <balaji.varadarajan@robinhood.com>
This commit is contained in:
Pratyaksh Sharma
2020-10-07 09:04:03 +05:30
committed by GitHub
parent fed01cd3c9
commit 524193eb4b
9 changed files with 252 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkCompactor.java
View File

@@ -41,7 +41,8 @@ public class HoodieSparkCompactor<T extends HoodieRecordPayload> extends Abstrac
@Override
public void compact(HoodieInstant instant) throws IOException {
LOG.info("Compactor executing compaction " + instant);
JavaRDD<WriteStatus> res = compactionClient.compact(instant.getTimestamp());
SparkRDDWriteClient<T> writeClient = (SparkRDDWriteClient<T>)compactionClient;
JavaRDD<WriteStatus> res = writeClient.compact(instant.getTimestamp());
long numWriteErrors = res.collect().stream().filter(WriteStatus::hasErrors).count();
if (numWriteErrors != 0) {
// We treat even a single error in compaction as fatal
@@ -50,6 +51,6 @@ public class HoodieSparkCompactor<T extends HoodieRecordPayload> extends Abstrac
"Compaction for instant (" + instant + ") failed with write errors. Errors :" + numWriteErrors);
}
// Commit compaction
compactionClient.commitCompaction(instant.getTimestamp(), res, Option.empty());
writeClient.commitCompaction(instant.getTimestamp(), res, Option.empty());
}
}