lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1200] fixed NPE in CustomKeyGenerator (#2093)

Browse Source 
- config field is no longer transient in key generator
- verified that the key generator object is shipped from the driver to executors, just the one time and reused for each record
This commit is contained in:
Pratyaksh Sharma
2020-10-21 12:06:25 +05:30
committed by GitHub
parent af5ef4d49d
commit e4931744eb
2 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java
View File

@@ -118,7 +118,6 @@ public class CustomKeyGenerator extends BuiltinKeyGenerator {
partitionPath.append(DEFAULT_PARTITION_PATH_SEPARATOR); partitionPath.append(DEFAULT_PARTITION_PATH_SEPARATOR);
} }
partitionPath.deleteCharAt(partitionPath.length() - 1); partitionPath.deleteCharAt(partitionPath.length() - 1);
return partitionPath.toString(); return partitionPath.toString();
} }

2
hudi-spark/src/main/java/org/apache/hudi/keygen/KeyGenerator.java
View File

@@ -41,7 +41,7 @@ public abstract class KeyGenerator implements Serializable, SparkKeyGeneratorInt
private static final String STRUCT_NAME = "hoodieRowTopLevelField"; private static final String STRUCT_NAME = "hoodieRowTopLevelField";
private static final String NAMESPACE = "hoodieRow"; private static final String NAMESPACE = "hoodieRow";
protected transient TypedProperties config; protected TypedProperties config;
private transient Function1<Object, Object> converterFn = null; private transient Function1<Object, Object> converterFn = null;
protected KeyGenerator(TypedProperties config) { protected KeyGenerator(TypedProperties config) {