[HUDI-1217] Improve avroToBytes method of HoodieAvroUtils (#2018)
This commit is contained in:
@@ -83,15 +83,17 @@ public class HoodieAvroUtils {
|
|||||||
/**
|
/**
|
||||||
* Convert a given avro record to bytes.
|
* Convert a given avro record to bytes.
|
||||||
*/
|
*/
|
||||||
public static byte[] avroToBytes(GenericRecord record) throws IOException {
|
public static byte[] avroToBytes(GenericRecord record) {
|
||||||
GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
|
GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
|
||||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||||
BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, reuseEncoder.get());
|
BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, reuseEncoder.get());
|
||||||
reuseEncoder.set(encoder);
|
reuseEncoder.set(encoder);
|
||||||
writer.write(record, encoder);
|
writer.write(record, encoder);
|
||||||
encoder.flush();
|
encoder.flush();
|
||||||
out.close();
|
|
||||||
return out.toByteArray();
|
return out.toByteArray();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new HoodieIOException("Cannot convert GenericRecord to bytes", e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -20,11 +20,9 @@ package org.apache.hudi.common.model;
|
|||||||
|
|
||||||
import org.apache.hudi.avro.HoodieAvroUtils;
|
import org.apache.hudi.avro.HoodieAvroUtils;
|
||||||
import org.apache.hudi.exception.HoodieException;
|
import org.apache.hudi.exception.HoodieException;
|
||||||
import org.apache.hudi.exception.HoodieIOException;
|
|
||||||
|
|
||||||
import org.apache.avro.generic.GenericRecord;
|
import org.apache.avro.generic.GenericRecord;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -48,11 +46,7 @@ public abstract class BaseAvroPayload implements Serializable {
|
|||||||
* @param orderingVal {@link Comparable} to be used in pre combine.
|
* @param orderingVal {@link Comparable} to be used in pre combine.
|
||||||
*/
|
*/
|
||||||
public BaseAvroPayload(GenericRecord record, Comparable orderingVal) {
|
public BaseAvroPayload(GenericRecord record, Comparable orderingVal) {
|
||||||
try {
|
|
||||||
this.recordBytes = record != null ? HoodieAvroUtils.avroToBytes(record) : new byte[0];
|
this.recordBytes = record != null ? HoodieAvroUtils.avroToBytes(record) : new byte[0];
|
||||||
} catch (IOException io) {
|
|
||||||
throw new HoodieIOException("Cannot convert GenericRecord to bytes", io);
|
|
||||||
}
|
|
||||||
this.orderingVal = orderingVal;
|
this.orderingVal = orderingVal;
|
||||||
if (orderingVal == null) {
|
if (orderingVal == null) {
|
||||||
throw new HoodieException("Ordering value is null for record: " + record);
|
throw new HoodieException("Ordering value is null for record: " + record);
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ package org.apache.hudi.common.model;
|
|||||||
|
|
||||||
import org.apache.hudi.avro.HoodieAvroUtils;
|
import org.apache.hudi.avro.HoodieAvroUtils;
|
||||||
import org.apache.hudi.common.util.Option;
|
import org.apache.hudi.common.util.Option;
|
||||||
import org.apache.hudi.exception.HoodieIOException;
|
|
||||||
|
|
||||||
import org.apache.avro.Schema;
|
import org.apache.avro.Schema;
|
||||||
import org.apache.avro.generic.GenericRecord;
|
import org.apache.avro.generic.GenericRecord;
|
||||||
@@ -39,15 +38,11 @@ public class HoodieAvroPayload implements HoodieRecordPayload<HoodieAvroPayload>
|
|||||||
private final byte[] recordBytes;
|
private final byte[] recordBytes;
|
||||||
|
|
||||||
public HoodieAvroPayload(Option<GenericRecord> record) {
|
public HoodieAvroPayload(Option<GenericRecord> record) {
|
||||||
try {
|
|
||||||
if (record.isPresent()) {
|
if (record.isPresent()) {
|
||||||
this.recordBytes = HoodieAvroUtils.avroToBytes(record.get());
|
this.recordBytes = HoodieAvroUtils.avroToBytes(record.get());
|
||||||
} else {
|
} else {
|
||||||
this.recordBytes = new byte[0];
|
this.recordBytes = new byte[0];
|
||||||
}
|
}
|
||||||
} catch (IOException io) {
|
|
||||||
throw new HoodieIOException("Cannot convert record to bytes", io);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ package org.apache.hudi.common.testutils;
|
|||||||
import org.apache.hudi.avro.HoodieAvroUtils;
|
import org.apache.hudi.avro.HoodieAvroUtils;
|
||||||
import org.apache.hudi.common.model.HoodieRecordPayload;
|
import org.apache.hudi.common.model.HoodieRecordPayload;
|
||||||
import org.apache.hudi.common.util.Option;
|
import org.apache.hudi.common.util.Option;
|
||||||
import org.apache.hudi.exception.HoodieIOException;
|
|
||||||
|
|
||||||
import org.apache.avro.Schema;
|
import org.apache.avro.Schema;
|
||||||
import org.apache.avro.generic.GenericRecord;
|
import org.apache.avro.generic.GenericRecord;
|
||||||
@@ -37,16 +36,11 @@ public class AvroBinaryTestPayload implements HoodieRecordPayload {
|
|||||||
private final byte[] recordBytes;
|
private final byte[] recordBytes;
|
||||||
|
|
||||||
public AvroBinaryTestPayload(Option<GenericRecord> record) {
|
public AvroBinaryTestPayload(Option<GenericRecord> record) {
|
||||||
|
|
||||||
try {
|
|
||||||
if (record.isPresent()) {
|
if (record.isPresent()) {
|
||||||
recordBytes = HoodieAvroUtils.avroToBytes(record.get());
|
recordBytes = HoodieAvroUtils.avroToBytes(record.get());
|
||||||
} else {
|
} else {
|
||||||
recordBytes = new byte[0];
|
recordBytes = new byte[0];
|
||||||
}
|
}
|
||||||
} catch (IOException io) {
|
|
||||||
throw new HoodieIOException("unable to convert payload to bytes");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
Reference in New Issue
Block a user