lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Close Hoodie Clients which are opened to properly shutdown embedded timeline service

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-06-11 12:58:58 -07:00
committed by vinoth chandar
parent 065173211e
commit 51d122b5c3
2 changed files with 52 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java
View File

@@ -183,14 +183,19 @@ public class DataSourceUtils {
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc,
JavaRDD<HoodieRecord> incomingHoodieRecords, JavaRDD<HoodieRecord> incomingHoodieRecords,
HoodieWriteConfig writeConfig) throws Exception { HoodieWriteConfig writeConfig) throws Exception {
HoodieReadClient client = null;
try { try {
HoodieReadClient client = new HoodieReadClient<>(jssc, writeConfig); client = new HoodieReadClient<>(jssc, writeConfig);
return client.tagLocation(incomingHoodieRecords) return client.tagLocation(incomingHoodieRecords)
.filter(r -> !((HoodieRecord<HoodieRecordPayload>) r).isCurrentLocationKnown()); .filter(r -> !((HoodieRecord<HoodieRecordPayload>) r).isCurrentLocationKnown());
} catch (DatasetNotFoundException e) { } catch (DatasetNotFoundException e) {
// this will be executed when there is no hoodie dataset yet // this will be executed when there is no hoodie dataset yet
// so no dups to drop // so no dups to drop
return incomingHoodieRecords; return incomingHoodieRecords;
} finally {
if (null != client) {
client.close();
}
} }
} }

9
hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCompactionAdminTool.java
View File

@@ -58,7 +58,8 @@ public class HoodieCompactionAdminTool {
*/ */
public void run(JavaSparkContext jsc) throws Exception { public void run(JavaSparkContext jsc) throws Exception {
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.basePath); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.basePath);
CompactionAdminClient admin = new CompactionAdminClient(jsc, cfg.basePath); final CompactionAdminClient admin = new CompactionAdminClient(jsc, cfg.basePath);
try {
final FileSystem fs = FSUtils.getFs(cfg.basePath, jsc.hadoopConfiguration()); final FileSystem fs = FSUtils.getFs(cfg.basePath, jsc.hadoopConfiguration());
if (cfg.outputPath != null && fs.exists(new Path(cfg.outputPath))) { if (cfg.outputPath != null && fs.exists(new Path(cfg.outputPath))) {
throw new IllegalStateException("Output File Path already exists"); throw new IllegalStateException("Output File Path already exists");
@@ -83,7 +84,8 @@ public class HoodieCompactionAdminTool {
break; break;
case UNSCHEDULE_PLAN: case UNSCHEDULE_PLAN:
List<RenameOpResult> r2 = List<RenameOpResult> r2 =
admin.unscheduleCompactionPlan(cfg.compactionInstantTime, cfg.skipValidation, cfg.parallelism, cfg.dryRun); admin
.unscheduleCompactionPlan(cfg.compactionInstantTime, cfg.skipValidation, cfg.parallelism, cfg.dryRun);
if (cfg.printOutput) { if (cfg.printOutput) {
printOperationResult("Result of Unscheduling Compaction Plan :", r2); printOperationResult("Result of Unscheduling Compaction Plan :", r2);
} }
@@ -100,6 +102,9 @@ public class HoodieCompactionAdminTool {
default: default:
throw new IllegalStateException("Not yet implemented !!"); throw new IllegalStateException("Not yet implemented !!");
} }
} finally {
admin.close();
}
} }
private <T> void serializeOperationResult(FileSystem fs, T result) throws Exception { private <T> void serializeOperationResult(FileSystem fs, T result) throws Exception {