1
0

[HUDI-3215] Solve UT for Spark 3.2 (#4565)

This commit is contained in:
Yann Byron
2022-01-27 06:48:26 +08:00
committed by GitHub
parent 3f21e5f14c
commit 4a9f826382
5 changed files with 78 additions and 37 deletions

View File

@@ -18,6 +18,7 @@
package org.apache.hudi.common.model;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
@@ -30,9 +31,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import static org.apache.hudi.avro.HoodieAvroUtils.bytesToAvro;
import static org.apache.hudi.avro.HoodieAvroUtils.getNestedFieldVal;
/**
* {@link HoodieRecordPayload} impl that honors ordering field in both preCombine and combineAndGetUpdateValue.
* <p>
@@ -57,7 +55,7 @@ public class DefaultHoodieRecordPayload extends OverwriteWithLatestAvroPayload {
return Option.empty();
}
GenericRecord incomingRecord = bytesToAvro(recordBytes, schema);
GenericRecord incomingRecord = HoodieAvroUtils.bytesToAvro(recordBytes, schema);
// Null check is needed here to support schema evolution. The record in storage may be from old schema where
// the new ordering column might not be present and hence returns null.
@@ -81,7 +79,7 @@ public class DefaultHoodieRecordPayload extends OverwriteWithLatestAvroPayload {
if (recordBytes.length == 0) {
return Option.empty();
}
GenericRecord incomingRecord = bytesToAvro(recordBytes, schema);
GenericRecord incomingRecord = HoodieAvroUtils.bytesToAvro(recordBytes, schema);
eventTime = updateEventTime(incomingRecord, properties);
return isDeleteRecord(incomingRecord) ? Option.empty() : Option.of(incomingRecord);
@@ -123,4 +121,24 @@ public class DefaultHoodieRecordPayload extends OverwriteWithLatestAvroPayload {
properties.getProperty(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP_KEY), true, consistentLogicalTimestampEnabled);
return persistedOrderingVal == null || ((Comparable) persistedOrderingVal).compareTo(incomingOrderingVal) <= 0;
}
/**
* a wrapper of HoodieAvroUtils.getNestedFieldVal.
* Within it, catch exceptions and return null when "returnNullIfNotFound" is true and can't take effect.
*/
private static Object getNestedFieldVal(
GenericRecord record,
String fieldName,
boolean returnNullIfNotFound,
boolean consistentLogicalTimestampEnabled) {
try {
return HoodieAvroUtils.getNestedFieldVal(record, fieldName, returnNullIfNotFound, consistentLogicalTimestampEnabled);
} catch (Exception e) {
if (returnNullIfNotFound) {
return null;
} else {
throw e;
}
}
}
}