1
0

Changing Update record failure semantics to be consistent with inserts

- Don't skip, but writes the same old record again now
 - Marks the correspoinding HoodieRecord as failure to be handed back to the client
This commit is contained in:
Vinoth Chandar
2017-01-05 17:26:47 -08:00
parent 93de14f2cf
commit b2c958519e
3 changed files with 22 additions and 6 deletions

View File

@@ -90,6 +90,8 @@ public class HoodieInsertHandle<T extends HoodieRecordPayload> extends HoodieIOH
record.deflate();
recordsWritten++;
} catch (Throwable t) {
// Not throwing exception from here, since we don't want to fail the entire job
// for a single record
status.markFailure(record, t);
logger.error("Error writing record " + record, t);
}

View File

@@ -114,17 +114,19 @@ import java.util.Iterator;
}
private void writeUpdateRecord(HoodieRecord<T> hoodieRecord, IndexedRecord indexedRecord) {
private boolean writeUpdateRecord(HoodieRecord<T> hoodieRecord, IndexedRecord indexedRecord) {
try {
storageWriter.writeAvroWithMetadata(indexedRecord, hoodieRecord);
hoodieRecord.deflate();
writeStatus.markSuccess(hoodieRecord);
recordsWritten ++;
updatedRecordsWritten ++;
return true;
} catch (Exception e) {
logger.error("Error writing record "+ hoodieRecord, e);
writeStatus.markFailure(hoodieRecord, e);
}
return false;
}
/**
@@ -133,16 +135,27 @@ import java.util.Iterator;
public void write(GenericRecord oldRecord) {
String key = oldRecord.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
HoodieRecord<T> hoodieRecord = keyToNewRecords.get(key);
boolean copyOldRecord = true;
if (keyToNewRecords.containsKey(key)) {
try {
IndexedRecord avroRecord = hoodieRecord.getData().combineAndGetUpdateValue(oldRecord, schema);
writeUpdateRecord(hoodieRecord, avroRecord);
if (writeUpdateRecord(hoodieRecord, avroRecord)) {
/* ONLY WHEN
* 1) we have an update for this key AND
* 2) We are able to successfully write the the combined new value
*
* We no longer need to copy the old record over.
*/
copyOldRecord = false;
}
keyToNewRecords.remove(key);
} catch (Exception e) {
throw new HoodieUpsertException("Failed to combine/merge new record with old value in storage, for new record {"
+ keyToNewRecords.get(key) + "}, old value {" + oldRecord + "}", e);
}
} else {
}
if (copyOldRecord) {
// this should work as it is, since this is an existing record
String errMsg = "Failed to merge old record into new file for key " + key + " from old file "
+ getOldFilePath() + " to new file " + newFilePath;