1
0

[HUDI-1217] Improve avroToBytes method of HoodieAvroUtils (#2018)

This commit is contained in:
Mathieu
2020-08-24 17:33:28 +08:00
committed by GitHub
parent 35b21855da
commit f8dcd5334e
4 changed files with 19 additions and 34 deletions

View File

@@ -83,15 +83,17 @@ public class HoodieAvroUtils {
/** /**
* Convert a given avro record to bytes. * Convert a given avro record to bytes.
*/ */
public static byte[] avroToBytes(GenericRecord record) throws IOException { public static byte[] avroToBytes(GenericRecord record) {
GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema()); GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
ByteArrayOutputStream out = new ByteArrayOutputStream(); try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, reuseEncoder.get()); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, reuseEncoder.get());
reuseEncoder.set(encoder); reuseEncoder.set(encoder);
writer.write(record, encoder); writer.write(record, encoder);
encoder.flush(); encoder.flush();
out.close();
return out.toByteArray(); return out.toByteArray();
} catch (IOException e) {
throw new HoodieIOException("Cannot convert GenericRecord to bytes", e);
}
} }
/** /**

View File

@@ -20,11 +20,9 @@ package org.apache.hudi.common.model;
import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
/** /**
@@ -48,11 +46,7 @@ public abstract class BaseAvroPayload implements Serializable {
* @param orderingVal {@link Comparable} to be used in pre combine. * @param orderingVal {@link Comparable} to be used in pre combine.
*/ */
public BaseAvroPayload(GenericRecord record, Comparable orderingVal) { public BaseAvroPayload(GenericRecord record, Comparable orderingVal) {
try {
this.recordBytes = record != null ? HoodieAvroUtils.avroToBytes(record) : new byte[0]; this.recordBytes = record != null ? HoodieAvroUtils.avroToBytes(record) : new byte[0];
} catch (IOException io) {
throw new HoodieIOException("Cannot convert GenericRecord to bytes", io);
}
this.orderingVal = orderingVal; this.orderingVal = orderingVal;
if (orderingVal == null) { if (orderingVal == null) {
throw new HoodieException("Ordering value is null for record: " + record); throw new HoodieException("Ordering value is null for record: " + record);

View File

@@ -20,7 +20,6 @@ package org.apache.hudi.common.model;
import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
@@ -39,15 +38,11 @@ public class HoodieAvroPayload implements HoodieRecordPayload<HoodieAvroPayload>
private final byte[] recordBytes; private final byte[] recordBytes;
public HoodieAvroPayload(Option<GenericRecord> record) { public HoodieAvroPayload(Option<GenericRecord> record) {
try {
if (record.isPresent()) { if (record.isPresent()) {
this.recordBytes = HoodieAvroUtils.avroToBytes(record.get()); this.recordBytes = HoodieAvroUtils.avroToBytes(record.get());
} else { } else {
this.recordBytes = new byte[0]; this.recordBytes = new byte[0];
} }
} catch (IOException io) {
throw new HoodieIOException("Cannot convert record to bytes", io);
}
} }
@Override @Override

View File

@@ -21,7 +21,6 @@ package org.apache.hudi.common.testutils;
import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
@@ -37,16 +36,11 @@ public class AvroBinaryTestPayload implements HoodieRecordPayload {
private final byte[] recordBytes; private final byte[] recordBytes;
public AvroBinaryTestPayload(Option<GenericRecord> record) { public AvroBinaryTestPayload(Option<GenericRecord> record) {
try {
if (record.isPresent()) { if (record.isPresent()) {
recordBytes = HoodieAvroUtils.avroToBytes(record.get()); recordBytes = HoodieAvroUtils.avroToBytes(record.get());
} else { } else {
recordBytes = new byte[0]; recordBytes = new byte[0];
} }
} catch (IOException io) {
throw new HoodieIOException("unable to convert payload to bytes");
}
} }
@Override @Override