lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Add nested fields support for MOR tables

Browse Source
This commit is contained in:
Jian Xu
2017-06-30 17:03:44 -07:00
committed by vinoth chandar
parent 6a3c94aaa3
commit b1cf097b0c
6 changed files with 268 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReader.java
View File

@@ -55,6 +55,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import parquet.avro.AvroSchemaConverter;
import parquet.hadoop.ParquetFileReader;
import parquet.schema.MessageType;
@@ -250,15 +252,25 @@ public class HoodieRealtimeRecordReader implements RecordReader<Void, ArrayWrita
case ENUM:
return new Text(value.toString());
case ARRAY:
Writable[] values2 = new Writable[schema.getFields().size()];
GenericArray arrayValue = (GenericArray) value;
Writable[] values2 = new Writable[arrayValue.size()];
int index2 = 0;
for (Object obj : (GenericArray) value) {
for (Object obj : arrayValue) {
values2[index2++] = avroToArrayWritable(obj, schema.getElementType());
}
return new ArrayWritable(Writable.class, values2);
case MAP:
// TODO(vc): Need to add support for complex types
return NullWritable.get();
Map mapValue = (Map) value;
Writable[] values3 = new Writable[mapValue.size()];
int index3 = 0;
for (Object entry : mapValue.entrySet()) {
Map.Entry mapEntry = (Map.Entry) entry;
Writable[] mapValues = new Writable[2];
mapValues[0] = new Text(mapEntry.getKey().toString());
mapValues[1] = avroToArrayWritable(mapEntry.getValue(), schema.getValueType());
values3[index3++] = new ArrayWritable(Writable.class, mapValues);
}
return new ArrayWritable(Writable.class, values3);
case UNION:
List<Schema> types = schema.getTypes();
if (types.size() != 2) {