lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1226] Fix ComplexKeyGenerator for non-partitioned tables

Browse Source
This commit is contained in:
Satish Kotha
2020-08-25 13:37:48 -07:00
committed by n3nash
parent df8f099c99
commit f468c20c6c
3 changed files with 25 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
View File

@@ -37,9 +37,9 @@ public class ComplexKeyGenerator extends BuiltinKeyGenerator {
public ComplexKeyGenerator(TypedProperties props) {
super(props);
this.recordKeyFields = Arrays.stream(props.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY())
.split(",")).map(String::trim).collect(Collectors.toList());
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
this.partitionPathFields = Arrays.stream(props.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY())
.split(",")).map(String::trim).collect(Collectors.toList());
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
}
@Override

4
hudi-spark/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java
View File

@@ -59,6 +59,10 @@ public class KeyGenUtils {
public static String getRecordPartitionPath(GenericRecord record, List<String> partitionPathFields,
boolean hiveStylePartitioning, boolean encodePartitionPath) {
if (partitionPathFields.isEmpty()) {
return "";
}
StringBuilder partitionPath = new StringBuilder();
for (String partitionPathField : partitionPathFields) {
String fieldVal = HoodieAvroUtils.getNestedFieldValAsString(record, partitionPathField, true);