lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Checking storage level before persisting preppedRecords

Browse Source
This commit is contained in:
Jian Xu
2018-03-20 12:06:15 -07:00
committed by vinoth chandar
parent 291a88ba94
commit 48643795b8
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
View File

@@ -421,7 +421,12 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> implements Seriali
final boolean isUpsert) { final boolean isUpsert) {
// Cache the tagged records, so we don't end up computing both // Cache the tagged records, so we don't end up computing both
preppedRecords.persist(StorageLevel.MEMORY_AND_DISK_SER()); // TODO: Consistent contract in HoodieWriteClient regarding preppedRecord storage level handling
if (preppedRecords.getStorageLevel() == StorageLevel.NONE()) {
preppedRecords.persist(StorageLevel.MEMORY_AND_DISK_SER());
} else {
logger.info("RDD PreppedRecords was persisted at: " + preppedRecords.getStorageLevel());
}
WorkloadProfile profile = null; WorkloadProfile profile = null;
if (hoodieTable.isWorkloadProfileNeeded()) { if (hoodieTable.isWorkloadProfileNeeded()) {