lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Close Hoodie Clients which are opened to properly shutdown embedded timeline service

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-06-11 12:58:58 -07:00
committed by vinoth chandar
parent 065173211e
commit 51d122b5c3
2 changed files with 52 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java
View File

@@ -183,14 +183,19 @@ public class DataSourceUtils {
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc,
JavaRDD<HoodieRecord> incomingHoodieRecords,
HoodieWriteConfig writeConfig) throws Exception {
HoodieReadClient client = null;
try {
HoodieReadClient client = new HoodieReadClient<>(jssc, writeConfig);
client = new HoodieReadClient<>(jssc, writeConfig);
return client.tagLocation(incomingHoodieRecords)
.filter(r -> !((HoodieRecord<HoodieRecordPayload>) r).isCurrentLocationKnown());
} catch (DatasetNotFoundException e) {
// this will be executed when there is no hoodie dataset yet
// so no dups to drop
return incomingHoodieRecords;
} finally {
if (null != client) {
client.close();
}
}
}