lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-757] Added hudi-cli command to export metadata of Instants.

Browse Source 
Example:
hudi:db.table-> export instants --localFolder /tmp/ --limit 5 --actions clean,rollback,commit --desc false
This commit is contained in:
Prashant Wason
2020-03-31 22:32:13 -07:00
committed by n3nash
parent 84dd9047d3
commit 62bd3e7ded
3 changed files with 264 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
View File

@@ -30,8 +30,11 @@ import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.io.JsonDecoder;
import org.apache.avro.io.JsonEncoder;
import org.codehaus.jackson.node.NullNode;
import java.io.ByteArrayInputStream;
@@ -79,6 +82,22 @@ public class HoodieAvroUtils {
return out.toByteArray();
}
/**
* Convert a given avro record to json and return the encoded bytes.
*
* @param record The GenericRecord to convert
* @param pretty Whether to pretty-print the json output
*/
public static byte[] avroToJson(GenericRecord record, boolean pretty) throws IOException {
DatumWriter<Object> writer = new GenericDatumWriter<>(record.getSchema());
ByteArrayOutputStream out = new ByteArrayOutputStream();
JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(record.getSchema(), out, pretty);
writer.write(record, jsonEncoder);
jsonEncoder.flush();
return out.toByteArray();
//metadata.toJsonString().getBytes(StandardCharsets.UTF_8));
}
/**
* Convert serialized bytes back into avro record.
*/
@@ -89,6 +108,16 @@ public class HoodieAvroUtils {
return reader.read(null, decoder);
}
/**
* Convert json bytes back into avro record.
*/
public static GenericRecord jsonBytesToAvro(byte[] bytes, Schema schema) throws IOException {
ByteArrayInputStream bio = new ByteArrayInputStream(bytes);
JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema, bio);
GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
return reader.read(null, jsonDecoder);
}
public static boolean isMetadataField(String fieldName) {
return HoodieRecord.COMMIT_TIME_METADATA_FIELD.equals(fieldName)
|| HoodieRecord.COMMIT_SEQNO_METADATA_FIELD.equals(fieldName)

4
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java
View File

@@ -132,6 +132,10 @@ public class TimelineMetadataUtils {
return deserializeAvroMetadata(bytes, HoodieCleanMetadata.class);
}
public static HoodieRollbackMetadata deserializeHoodieRollbackMetadata(byte[] bytes) throws IOException {
return deserializeAvroMetadata(bytes, HoodieRollbackMetadata.class);
}
public static HoodieSavepointMetadata deserializeHoodieSavepointMetadata(byte[] bytes) throws IOException {
return deserializeAvroMetadata(bytes, HoodieSavepointMetadata.class);
}