lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2073] Fix the bug of hoodieClusteringJob never quit (#3157)

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
zhangyue19921010
2021-06-27 13:03:41 +08:00
committed by GitHub
parent f73bedd374
commit e99a6b031b
1 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
View File

@@ -149,11 +149,11 @@ public class HoodieClusteringJob {
private int doCluster(JavaSparkContext jsc) throws Exception { private int doCluster(JavaSparkContext jsc) throws Exception {
String schemaStr = getSchemaFromLatestInstant(); String schemaStr = getSchemaFromLatestInstant();
SparkRDDWriteClient<HoodieRecordPayload> client = try (SparkRDDWriteClient<HoodieRecordPayload> client = UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) {
UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props); JavaRDD<WriteStatus> writeResponse =
JavaRDD<WriteStatus> writeResponse = client.cluster(cfg.clusteringInstantTime, true).getWriteStatuses();
client.cluster(cfg.clusteringInstantTime, true).getWriteStatuses(); return UtilHelpers.handleErrors(jsc, cfg.clusteringInstantTime, writeResponse);
return UtilHelpers.handleErrors(jsc, cfg.clusteringInstantTime, writeResponse); }
} }
@TestOnly @TestOnly
@@ -163,12 +163,12 @@ public class HoodieClusteringJob {
private Option<String> doSchedule(JavaSparkContext jsc) throws Exception { private Option<String> doSchedule(JavaSparkContext jsc) throws Exception {
String schemaStr = getSchemaFromLatestInstant(); String schemaStr = getSchemaFromLatestInstant();
SparkRDDWriteClient<HoodieRecordPayload> client = try (SparkRDDWriteClient<HoodieRecordPayload> client = UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props)) {
UtilHelpers.createHoodieClient(jsc, cfg.basePath, schemaStr, cfg.parallelism, Option.empty(), props); if (cfg.clusteringInstantTime != null) {
if (cfg.clusteringInstantTime != null) { client.scheduleClusteringAtInstant(cfg.clusteringInstantTime, Option.empty());
client.scheduleClusteringAtInstant(cfg.clusteringInstantTime, Option.empty()); return Option.of(cfg.clusteringInstantTime);
return Option.of(cfg.clusteringInstantTime); }
return client.scheduleClustering(Option.empty());
} }
return client.scheduleClustering(Option.empty());
} }
} }