lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3640] Set SimpleKeyGenerator as default in 2to3 table upgrade for Spark engine (#5075)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-03-21 17:35:06 -07:00
committed by GitHub
parent ca0931d332
commit 9b6e138af2
4 changed files with 104 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToThreeUpgradeHandler.java
View File

@@ -20,6 +20,7 @@
package org.apache.hudi.table.upgrade;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.util.Option;
@@ -34,6 +35,8 @@ import java.util.Map;
* UpgradeHandler to assist in upgrading {@link org.apache.hudi.table.HoodieTable} from version 2 to 3.
*/
public class TwoToThreeUpgradeHandler implements UpgradeHandler {
public static final String SPARK_SIMPLE_KEY_GENERATOR = "org.apache.hudi.keygen.SimpleKeyGenerator";
@Override
public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) {
if (config.isMetadataTableEnabled()) {
@@ -47,8 +50,13 @@ public class TwoToThreeUpgradeHandler implements UpgradeHandler {
tablePropsToAdd.put(HoodieTableConfig.HIVE_STYLE_PARTITIONING_ENABLE, config.getStringOrDefault(HoodieTableConfig.HIVE_STYLE_PARTITIONING_ENABLE));
String keyGenClassName = Option.ofNullable(config.getString(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME))
.orElse(config.getString(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME));
if (keyGenClassName == null && config.getEngineType() == EngineType.SPARK) {
// For Spark, if the key generator class is not configured by user,
// set it to SimpleKeyGenerator as default
keyGenClassName = SPARK_SIMPLE_KEY_GENERATOR;
}
ValidationUtils.checkState(keyGenClassName != null, String.format("Missing config: %s or %s",
HoodieTableConfig.KEY_GENERATOR_CLASS_NAME, HoodieWriteConfig.KEYGENERATOR_CLASS_NAME));
HoodieTableConfig.KEY_GENERATOR_CLASS_NAME, HoodieWriteConfig.KEYGENERATOR_CLASS_NAME));
tablePropsToAdd.put(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME, keyGenClassName);
return tablePropsToAdd;
}