lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2073] Fix the bug of hoodieClusteringJob never quit (#3157)

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
zhangyue19921010
2021-06-27 13:03:41 +08:00
committed by GitHub
parent f73bedd374
commit e99a6b031b
1 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
View File

@@ -149,12 +149,12 @@ public class HoodieClusteringJob {
private int doCluster(JavaSparkContext jsc) throws Exception { private int doCluster(JavaSparkContext jsc) throws Exception {
String schemaStr = getSchemaFromLatestInstant(); String schemaStr = getSchemaFromLatestInstant();
SparkRDDWriteClient<HoodieRecordPayload> client = try (SparkRDDWriteClient<HoodieRecordPayload> client = UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) {
UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props);
JavaRDD<WriteStatus> writeResponse = JavaRDD<WriteStatus> writeResponse =
client.cluster(cfg.clusteringInstantTime, true).getWriteStatuses(); client.cluster(cfg.clusteringInstantTime, true).getWriteStatuses();
return UtilHelpers.handleErrors(jsc, cfg.clusteringInstantTime, writeResponse); return UtilHelpers.handleErrors(jsc, cfg.clusteringInstantTime, writeResponse);
} }
}
@TestOnly @TestOnly
public Option<String> doSchedule() throws Exception { public Option<String> doSchedule() throws Exception {
@@ -163,12 +163,12 @@ public class HoodieClusteringJob {
private Option<String> doSchedule(JavaSparkContext jsc) throws Exception { private Option<String> doSchedule(JavaSparkContext jsc) throws Exception {
String schemaStr = getSchemaFromLatestInstant(); String schemaStr = getSchemaFromLatestInstant();
SparkRDDWriteClient<HoodieRecordPayload> client = try (SparkRDDWriteClient<HoodieRecordPayload> client = UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) {
UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props);
if (cfg.clusteringInstantTime != null) { if (cfg.clusteringInstantTime != null) {
client.scheduleClusteringAtInstant(cfg.clusteringInstantTime, Option.empty()); client.scheduleClusteringAtInstant(cfg.clusteringInstantTime, Option.empty());
return Option.of(cfg.clusteringInstantTime); return Option.of(cfg.clusteringInstantTime);
} }
return client.scheduleClustering(Option.empty()); return client.scheduleClustering(Option.empty());
} }
}
} }