lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-327] Add null/empty checks to key generators (#1040)

Browse Source 
* Adds null and empty checks to all key generators. 
* Also improves error messaging for key generator issues.
This commit is contained in:
bschell
2019-11-26 02:37:16 -08:00
committed by vinoth chandar
parent 2a4cfb47c7
commit 60fed21dc7
7 changed files with 161 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java
View File

@@ -30,6 +30,7 @@ import org.apache.hudi.DataSourceUtils;
import org.apache.hudi.SimpleKeyGenerator;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.util.TypedProperties;
import org.apache.hudi.exception.HoodieKeyException;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.utilities.exception.HoodieDeltaStreamerException;
@@ -98,8 +99,11 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
}
Date timestamp = this.timestampType == TimestampType.EPOCHMILLISECONDS ? new Date(unixTime) : new Date(unixTime * 1000);
return new HoodieKey(DataSourceUtils.getNestedFieldValAsString(record, recordKeyField),
partitionPathFormat.format(timestamp));
String recordKey = DataSourceUtils.getNullableNestedFieldValAsString(record, recordKeyField);
if (recordKey == null || recordKey.isEmpty()) {
throw new HoodieKeyException("recordKey value: \"" + recordKey + "\" for field: \"" + recordKeyField + "\" cannot be null or empty.");
}
return new HoodieKey(recordKey, partitionPathFormat.format(timestamp));
} catch (ParseException pe) {
throw new HoodieDeltaStreamerException("Unable to parse input partition field :" + partitionVal, pe);
}