lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3936] Fix projection for a nested field as pre-combined key (#5379)

Browse Source 
This PR fixes the projection logic around a nested field which is used as the pre-combined key field. The fix is to only check and append the root level field for projection, i.e., "a", for a nested field "a.b.c" in the mandatory columns.

- Changes the logic to check and append the root level field for a required nested field in the mandatory columns in HoodieBaseRelation.appendMandatoryColumns
This commit is contained in:
Y Ethan Guo
2022-04-21 17:17:57 -07:00
committed by GitHub
parent 037f89ee7c
commit c4bc2deea0
8 changed files with 66 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
View File

@@ -18,6 +18,17 @@
package org.apache.hudi.avro;
import org.apache.hudi.common.config.SerializableSchema;
import org.apache.hudi.common.model.HoodieOperation;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.SchemaCompatibilityException;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Conversions;
import org.apache.avro.Conversions.DecimalConversion;
@@ -42,16 +53,6 @@ import org.apache.avro.io.EncoderFactory;
import org.apache.avro.io.JsonDecoder;
import org.apache.avro.io.JsonEncoder;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.hudi.common.config.SerializableSchema;
import org.apache.hudi.common.model.HoodieOperation;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.SchemaCompatibilityException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -480,6 +481,17 @@ public class HoodieAvroUtils {
return projectedSchema;
}
/**
* Obtain the root-level field name of a full field name, possibly a nested field.
* For example, given "a.b.c", the output is "a"; given "a", the output is "a".
*
* @param fieldName The field name.
* @return Root-level field name
*/
public static String getRootLevelFieldName(String fieldName) {
return fieldName.split("\\.")[0];
}
/**
* Obtain value of the provided field as string, denoted by dot notation. e.g: a.b.c
*/

7
hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
View File

@@ -257,6 +257,13 @@ public class TestHoodieAvroUtils {
assertEquals(expectedSchema, rec1.getSchema());
}
@Test
public void testGetRootLevelFieldName() {
assertEquals("a", HoodieAvroUtils.getRootLevelFieldName("a.b.c"));
assertEquals("a", HoodieAvroUtils.getRootLevelFieldName("a"));
assertEquals("", HoodieAvroUtils.getRootLevelFieldName(""));
}
@Test
public void testGetNestedFieldVal() {
GenericRecord rec = new GenericData.Record(new Schema.Parser().parse(EXAMPLE_SCHEMA));