lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Keep non-conflicting names for common configs between DataSourceOptions and HoodieWriteConfig (#3511)

Browse Source
This commit is contained in:
Udit Mehrotra
2021-08-20 02:42:59 -07:00
committed by GitHub
parent 49829f8822
commit e39d0a2f28
31 changed files with 126 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -84,7 +84,7 @@ public class HoodieWriteConfig extends HoodieConfig {
.noDefaultValue() .noDefaultValue()
.withDocumentation("Table name that will be used for registering with metastores like HMS. Needs to be same across runs."); .withDocumentation("Table name that will be used for registering with metastores like HMS. Needs to be same across runs.");
public static final ConfigProperty<String> PRECOMBINE_FIELD = ConfigProperty public static final ConfigProperty<String> PRECOMBINE_FIELD_NAME = ConfigProperty
.key("hoodie.datasource.write.precombine.field") .key("hoodie.datasource.write.precombine.field")
.defaultValue("ts") .defaultValue("ts")
.withDocumentation("Field used in preCombining before actual write. When two records have the same key value, " .withDocumentation("Field used in preCombining before actual write. When two records have the same key value, "
@@ -423,10 +423,10 @@ public class HoodieWriteConfig extends HoodieConfig {
@Deprecated @Deprecated
public static final String TABLE_NAME = TBL_NAME.key(); public static final String TABLE_NAME = TBL_NAME.key();
/** /**
* @deprecated Use {@link #PRECOMBINE_FIELD} and its methods instead * @deprecated Use {@link #PRECOMBINE_FIELD_NAME} and its methods instead
*/ */
@Deprecated @Deprecated
public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD.key(); public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD_NAME.key();
/** /**
* @deprecated Use {@link #WRITE_PAYLOAD_CLASS_NAME} and its methods instead * @deprecated Use {@link #WRITE_PAYLOAD_CLASS_NAME} and its methods instead
*/ */
@@ -859,7 +859,7 @@ public class HoodieWriteConfig extends HoodieConfig {
} }
public String getPreCombineField() { public String getPreCombineField() {
return getString(PRECOMBINE_FIELD); return getString(PRECOMBINE_FIELD_NAME);
} }
public String getWritePayloadClass() { public String getWritePayloadClass() {
@@ -1803,7 +1803,7 @@ public class HoodieWriteConfig extends HoodieConfig {
} }
public Builder withPreCombineField(String preCombineField) { public Builder withPreCombineField(String preCombineField) {
writeConfig.setValue(PRECOMBINE_FIELD, preCombineField); writeConfig.setValue(PRECOMBINE_FIELD_NAME, preCombineField);
return this; return this;
} }

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java
View File

@@ -32,9 +32,9 @@ public class ComplexAvroKeyGenerator extends BaseKeyGenerator {
public ComplexAvroKeyGenerator(TypedProperties props) { public ComplexAvroKeyGenerator(TypedProperties props) {
super(props); super(props);
this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
} }

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java
View File

@@ -55,8 +55,8 @@ public class CustomAvroKeyGenerator extends BaseKeyGenerator {
public CustomAvroKeyGenerator(TypedProperties props) { public CustomAvroKeyGenerator(TypedProperties props) {
super(props); super(props);
this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
} }
@Override @Override

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java
View File

@@ -35,7 +35,7 @@ public class GlobalAvroDeleteKeyGenerator extends BaseKeyGenerator {
public GlobalAvroDeleteKeyGenerator(TypedProperties config) { public GlobalAvroDeleteKeyGenerator(TypedProperties config) {
super(config); super(config);
this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")); this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(","));
} }
@Override @Override

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java
View File

@@ -36,7 +36,7 @@ public class NonpartitionedAvroKeyGenerator extends BaseKeyGenerator {
public NonpartitionedAvroKeyGenerator(TypedProperties props) { public NonpartitionedAvroKeyGenerator(TypedProperties props) {
super(props); super(props);
this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
this.partitionPathFields = EMPTY_PARTITION_FIELD_LIST; this.partitionPathFields = EMPTY_PARTITION_FIELD_LIST;
} }

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java
View File

@@ -29,8 +29,8 @@ import java.util.Collections;
public class SimpleAvroKeyGenerator extends BaseKeyGenerator { public class SimpleAvroKeyGenerator extends BaseKeyGenerator {
public SimpleAvroKeyGenerator(TypedProperties props) { public SimpleAvroKeyGenerator(TypedProperties props) {
this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
} }
SimpleAvroKeyGenerator(TypedProperties props, String partitionPathField) { SimpleAvroKeyGenerator(TypedProperties props, String partitionPathField) {

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
View File

@@ -88,8 +88,8 @@ public class TimestampBasedAvroKeyGenerator extends SimpleAvroKeyGenerator {
} }
public TimestampBasedAvroKeyGenerator(TypedProperties config) throws IOException { public TimestampBasedAvroKeyGenerator(TypedProperties config) throws IOException {
this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
} }
TimestampBasedAvroKeyGenerator(TypedProperties config, String partitionPathField) throws IOException { TimestampBasedAvroKeyGenerator(TypedProperties config, String partitionPathField) throws IOException {

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java
View File

@@ -33,7 +33,7 @@ public abstract class BaseOneToTwoUpgradeHandler implements UpgradeHandler {
public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime) { public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime) {
Map<ConfigProperty, String> tablePropsToAdd = new HashMap<>(); Map<ConfigProperty, String> tablePropsToAdd = new HashMap<>();
tablePropsToAdd.put(HoodieTableConfig.PARTITION_FIELDS, getPartitionColumns(config)); tablePropsToAdd.put(HoodieTableConfig.PARTITION_FIELDS, getPartitionColumns(config));
tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()));
tablePropsToAdd.put(HoodieTableConfig.BASE_FILE_FORMAT, config.getString(HoodieTableConfig.BASE_FILE_FORMAT)); tablePropsToAdd.put(HoodieTableConfig.BASE_FILE_FORMAT, config.getString(HoodieTableConfig.BASE_FILE_FORMAT));
return tablePropsToAdd; return tablePropsToAdd;
} }

6
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java
View File

@@ -52,9 +52,9 @@ public class TestCreateAvroKeyGeneratorByTypeWithFactory {
@BeforeEach @BeforeEach
public void init() { public void init() {
props = new TypedProperties(); props = new TypedProperties();
props.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); props.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
// for timestamp based key generator // for timestamp based key generator
props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING"); props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING");

6
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java
View File

@@ -67,9 +67,9 @@ public class TestHoodieAvroKeyGeneratorFactory {
private TypedProperties getCommonProps() { private TypedProperties getCommonProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
} }

2
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java
View File

@@ -25,6 +25,6 @@ public class OneToTwoUpgradeHandler extends BaseOneToTwoUpgradeHandler {
@Override @Override
String getPartitionColumns(HoodieWriteConfig config) { String getPartitionColumns(HoodieWriteConfig config) {
return config.getProps().getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()); return config.getProps().getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key());
} }
} }

4
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
View File

@@ -37,9 +37,9 @@ public class ComplexKeyGenerator extends BuiltinKeyGenerator {
public ComplexKeyGenerator(TypedProperties props) { public ComplexKeyGenerator(TypedProperties props) {
super(props); super(props);
this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
complexAvroKeyGenerator = new ComplexAvroKeyGenerator(props); complexAvroKeyGenerator = new ComplexAvroKeyGenerator(props);
} }

4
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java
View File

@@ -53,8 +53,8 @@ public class CustomKeyGenerator extends BuiltinKeyGenerator {
public CustomKeyGenerator(TypedProperties props) { public CustomKeyGenerator(TypedProperties props) {
super(props); super(props);
this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
customAvroKeyGenerator = new CustomAvroKeyGenerator(props); customAvroKeyGenerator = new CustomAvroKeyGenerator(props);
} }

2
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java
View File

@@ -39,7 +39,7 @@ public class GlobalDeleteKeyGenerator extends BuiltinKeyGenerator {
private final GlobalAvroDeleteKeyGenerator globalAvroDeleteKeyGenerator; private final GlobalAvroDeleteKeyGenerator globalAvroDeleteKeyGenerator;
public GlobalDeleteKeyGenerator(TypedProperties config) { public GlobalDeleteKeyGenerator(TypedProperties config) {
super(config); super(config);
this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")); this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(","));
globalAvroDeleteKeyGenerator = new GlobalAvroDeleteKeyGenerator(config); globalAvroDeleteKeyGenerator = new GlobalAvroDeleteKeyGenerator(config);
} }

2
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java
View File

@@ -39,7 +39,7 @@ public class NonpartitionedKeyGenerator extends BuiltinKeyGenerator {
public NonpartitionedKeyGenerator(TypedProperties props) { public NonpartitionedKeyGenerator(TypedProperties props) {
super(props); super(props);
this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).collect(Collectors.toList()); .split(",")).map(String::trim).collect(Collectors.toList());
this.partitionPathFields = Collections.emptyList(); this.partitionPathFields = Collections.emptyList();
nonpartitionedAvroKeyGenerator = new NonpartitionedAvroKeyGenerator(props); nonpartitionedAvroKeyGenerator = new NonpartitionedAvroKeyGenerator(props);

4
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java
View File

@@ -36,8 +36,8 @@ public class SimpleKeyGenerator extends BuiltinKeyGenerator {
private final SimpleAvroKeyGenerator simpleAvroKeyGenerator; private final SimpleAvroKeyGenerator simpleAvroKeyGenerator;
public SimpleKeyGenerator(TypedProperties props) { public SimpleKeyGenerator(TypedProperties props) {
this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
} }
SimpleKeyGenerator(TypedProperties props, String partitionPathField) { SimpleKeyGenerator(TypedProperties props, String partitionPathField) {

4
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java
View File

@@ -41,8 +41,8 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
private final TimestampBasedAvroKeyGenerator timestampBasedAvroKeyGenerator; private final TimestampBasedAvroKeyGenerator timestampBasedAvroKeyGenerator;
public TimestampBasedKeyGenerator(TypedProperties config) throws IOException { public TimestampBasedKeyGenerator(TypedProperties config) throws IOException {
this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
} }
TimestampBasedKeyGenerator(TypedProperties config, String partitionPathField) throws IOException { TimestampBasedKeyGenerator(TypedProperties config, String partitionPathField) throws IOException {

2
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
View File

@@ -258,7 +258,7 @@ object HoodieSparkUtils extends SparkAdapterSupport {
.mkString(",") .mkString(",")
case b: BaseKeyGenerator => b.getPartitionPathFields.asScala.mkString(",") case b: BaseKeyGenerator => b.getPartitionPathFields.asScala.mkString(",")
case _ => typedProperties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) case _ => typedProperties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
} }
} }

26
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java
View File

@@ -39,11 +39,11 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey) { private TypedProperties getCommonProps(boolean getComplexRecordKey) {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) { if (getComplexRecordKey) {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col");
} else { } else {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
} }
properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties; return properties;
} }
@@ -53,20 +53,20 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesWithoutRecordKeyProp() { private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
private TypedProperties getWrongRecordKeyFieldProps() { private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties; return properties;
} }
private TypedProperties getProps() { private TypedProperties getProps() {
TypedProperties properties = getCommonProps(true); TypedProperties properties = getCommonProps(true);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms");
return properties; return properties;
} }
@@ -105,8 +105,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
@Test @Test
public void testSingleValueKeyGenerator() { public void testSingleValueKeyGenerator() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties); ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 1); assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 1);
assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 1); assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 1);
@@ -128,8 +128,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
@Test @Test
public void testMultipleValueKeyGenerator() { public void testMultipleValueKeyGenerator() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,timestamp"); properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp");
properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "rider,driver"); properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "rider,driver");
ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties); ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2); assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2);
assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 2); assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 2);
@@ -154,8 +154,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
@Test @Test
public void testMultipleValueKeyGeneratorNonPartitioned() { public void testMultipleValueKeyGeneratorNonPartitioned() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,timestamp"); properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp");
properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties); ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2); assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2);
assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 0); assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 0);

22
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
View File

@@ -48,53 +48,53 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey, boolean useKeyGeneratorClassName) { private TypedProperties getCommonProps(boolean getComplexRecordKey, boolean useKeyGeneratorClassName) {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) { if (getComplexRecordKey) {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col");
} else { } else {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
} }
if (useKeyGeneratorClassName) { if (useKeyGeneratorClassName) {
properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName()); properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName());
} else { } else {
properties.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name()); properties.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name());
} }
properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties; return properties;
} }
private TypedProperties getPropertiesForSimpleKeyGen(boolean useKeyGeneratorClassName) { private TypedProperties getPropertiesForSimpleKeyGen(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple");
return properties; return properties;
} }
private TypedProperties getImproperPartitionFieldFormatProp(boolean useKeyGeneratorClassName) { private TypedProperties getImproperPartitionFieldFormatProp(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
private TypedProperties getInvalidPartitionKeyTypeProps(boolean useKeyGeneratorClassName) { private TypedProperties getInvalidPartitionKeyTypeProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:dummy"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:dummy");
return properties; return properties;
} }
private TypedProperties getComplexRecordKeyWithSimplePartitionProps(boolean useKeyGeneratorClassName) { private TypedProperties getComplexRecordKeyWithSimplePartitionProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName); TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple");
return properties; return properties;
} }
private TypedProperties getComplexRecordKeyAndPartitionPathProps(boolean useKeyGeneratorClassName) { private TypedProperties getComplexRecordKeyAndPartitionPathProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName); TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple,ts_ms:timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple,ts_ms:timestamp");
populateNecessaryPropsForTimestampBasedKeyGen(properties); populateNecessaryPropsForTimestampBasedKeyGen(properties);
return properties; return properties;
} }
private TypedProperties getPropsWithoutRecordKeyFieldProps(boolean useKeyGeneratorClassName) { private TypedProperties getPropsWithoutRecordKeyFieldProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple");
if (useKeyGeneratorClassName) { if (useKeyGeneratorClassName) {
properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName()); properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName());
} else { } else {
@@ -111,14 +111,14 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesForTimestampBasedKeyGen(boolean useKeyGeneratorClassName) { private TypedProperties getPropertiesForTimestampBasedKeyGen(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "ts_ms:timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "ts_ms:timestamp");
populateNecessaryPropsForTimestampBasedKeyGen(properties); populateNecessaryPropsForTimestampBasedKeyGen(properties);
return properties; return properties;
} }
private TypedProperties getPropertiesForNonPartitionedKeyGen(boolean useKeyGeneratorClassName) { private TypedProperties getPropertiesForNonPartitionedKeyGen(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
return properties; return properties;
} }

12
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java
View File

@@ -34,29 +34,29 @@ public class TestGlobalDeleteKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey) { private TypedProperties getCommonProps(boolean getComplexRecordKey) {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) { if (getComplexRecordKey) {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,pii_col"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,pii_col");
} else { } else {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
} }
properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties; return properties;
} }
private TypedProperties getPropertiesWithoutRecordKeyProp() { private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
private TypedProperties getWrongRecordKeyFieldProps() { private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties; return properties;
} }
private TypedProperties getProps() { private TypedProperties getProps() {
TypedProperties properties = getCommonProps(true); TypedProperties properties = getCommonProps(true);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms");
return properties; return properties;
} }

22
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java
View File

@@ -37,11 +37,11 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey) { private TypedProperties getCommonProps(boolean getComplexRecordKey) {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) { if (getComplexRecordKey) {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col");
} else { } else {
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
} }
properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties; return properties;
} }
@@ -51,19 +51,19 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesWithPartitionPathProp() { private TypedProperties getPropertiesWithPartitionPathProp() {
TypedProperties properties = getCommonProps(true); TypedProperties properties = getCommonProps(true);
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms");
return properties; return properties;
} }
private TypedProperties getPropertiesWithoutRecordKeyProp() { private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
private TypedProperties getWrongRecordKeyFieldProps() { private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties; return properties;
} }
@@ -78,7 +78,7 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties); NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties);
GenericRecord record = getRecord(); GenericRecord record = getRecord();
Row row = KeyGeneratorTestUtilities.getRow(record); Row row = KeyGeneratorTestUtilities.getRow(record);
Assertions.assertEquals(properties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()), "timestamp,ts_ms"); Assertions.assertEquals(properties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()), "timestamp,ts_ms");
Assertions.assertEquals(keyGenerator.getPartitionPath(row), ""); Assertions.assertEquals(keyGenerator.getPartitionPath(row), "");
} }
@@ -101,8 +101,8 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
@Test @Test
public void testSingleValueKeyGeneratorNonPartitioned() { public void testSingleValueKeyGeneratorNonPartitioned() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "timestamp"); properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "timestamp");
properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties); NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties);
assertEquals(keyGenerator.getRecordKeyFields().size(), 1); assertEquals(keyGenerator.getRecordKeyFields().size(), 1);
assertEquals(keyGenerator.getPartitionPathFields().size(), 0); assertEquals(keyGenerator.getPartitionPathFields().size(), 0);
@@ -118,8 +118,8 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
@Test @Test
public void testMultipleValueKeyGeneratorNonPartitioned1() { public void testMultipleValueKeyGeneratorNonPartitioned1() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "timestamp,driver"); properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "timestamp,driver");
properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties); NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties);
assertEquals(keyGenerator.getRecordKeyFields().size(), 2); assertEquals(keyGenerator.getRecordKeyFields().size(), 2);
assertEquals(keyGenerator.getPartitionPathFields().size(), 0); assertEquals(keyGenerator.getPartitionPathFields().size(), 0);

22
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java
View File

@@ -39,8 +39,8 @@ import static org.apache.hudi.keygen.KeyGenUtils.DEFAULT_PARTITION_PATH;
public class TestSimpleKeyGenerator extends KeyGeneratorTestUtilities { public class TestSimpleKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps() { private TypedProperties getCommonProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties; return properties;
} }
@@ -50,40 +50,40 @@ public class TestSimpleKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesWithoutRecordKeyProp() { private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
private TypedProperties getWrongRecordKeyFieldProps() { private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties; return properties;
} }
private TypedProperties getWrongPartitionPathFieldProps() { private TypedProperties getWrongPartitionPathFieldProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "_wrong_partition_path"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "_wrong_partition_path");
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
return properties; return properties;
} }
private TypedProperties getComplexRecordKeyProp() { private TypedProperties getComplexRecordKeyProp() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,pii_col"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,pii_col");
return properties; return properties;
} }
private TypedProperties getProps() { private TypedProperties getProps() {
TypedProperties properties = getCommonProps(); TypedProperties properties = getCommonProps();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
private TypedProperties getPropsWithNestedPartitionPathField() { private TypedProperties getPropsWithNestedPartitionPathField() {
TypedProperties properties = getCommonProps(); TypedProperties properties = getCommonProps();
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "nested_col.prop1"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "nested_col.prop1");
return properties; return properties;
} }

6
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java
View File

@@ -64,9 +64,9 @@ public class TestTimestampBasedKeyGenerator {
baseRow = genericRecordToRow(baseRecord); baseRow = genericRecordToRow(baseRecord);
internalRow = KeyGeneratorTestUtilities.getInternalRow(baseRow); internalRow = KeyGeneratorTestUtilities.getInternalRow(baseRow);
properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "field1"); properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "field1");
properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "createTime"); properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "createTime");
properties.setProperty(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "false"); properties.setProperty(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "false");
} }
private TypedProperties getBaseKeyConfig(String timestampType, String dateFormat, String timezone, String scalarType) { private TypedProperties getBaseKeyConfig(String timestampType, String dateFormat, String timezone, String scalarType) {

6
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java
View File

@@ -53,9 +53,9 @@ public class TestCreateKeyGeneratorByTypeWithFactory {
@BeforeEach @BeforeEach
public void init() { public void init() {
props = new TypedProperties(); props = new TypedProperties();
props.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); props.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
// for timestamp based key generator // for timestamp based key generator
props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING"); props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING");

6
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java
View File

@@ -74,9 +74,9 @@ public class TestHoodieSparkKeyGeneratorFactory {
private TypedProperties getCommonProps() { private TypedProperties getCommonProps() {
TypedProperties properties = new TypedProperties(); TypedProperties properties = new TypedProperties();
properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties; return properties;
} }
} }

12
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
View File

@@ -231,8 +231,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
} }
private void addNewTableParamsToProps(Map<String, String> params) { private void addNewTableParamsToProps(Map<String, String> params) {
params.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "uuid"); params.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "uuid");
params.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "partition_path"); params.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path");
params.put(HoodieTableConfig.NAME.key(), metaClient.getTableConfig().getTableName()); params.put(HoodieTableConfig.NAME.key(), metaClient.getTableConfig().getTableName());
params.put(BASE_FILE_FORMAT.key(), BASE_FILE_FORMAT.defaultValue().name()); params.put(BASE_FILE_FORMAT.key(), BASE_FILE_FORMAT.defaultValue().name());
} }
@@ -262,8 +262,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
private void assertTableProps(HoodieWriteConfig cfg) { private void assertTableProps(HoodieWriteConfig cfg) {
HoodieTableConfig tableConfig = metaClient.getTableConfig(); HoodieTableConfig tableConfig = metaClient.getTableConfig();
Properties originalProps = cfg.getProps(); Properties originalProps = cfg.getProps();
assertEquals(tableConfig.getPartitionFieldProp(), originalProps.getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); assertEquals(tableConfig.getPartitionFieldProp(), originalProps.getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
assertEquals(tableConfig.getRecordKeyFieldProp(), originalProps.getProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key())); assertEquals(tableConfig.getRecordKeyFieldProp(), originalProps.getProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()));
assertEquals(tableConfig.getTableName(), cfg.getTableName()); assertEquals(tableConfig.getTableName(), cfg.getTableName());
assertEquals(tableConfig.getBaseFileFormat().name(), originalProps.getProperty(BASE_FILE_FORMAT.key())); assertEquals(tableConfig.getBaseFileFormat().name(), originalProps.getProperty(BASE_FILE_FORMAT.key()));
} }
@@ -289,8 +289,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
// set table configs // set table configs
HoodieTableConfig tableConfig = metaClient.getTableConfig(); HoodieTableConfig tableConfig = metaClient.getTableConfig();
tableConfig.setValue(HoodieTableConfig.NAME, cfg.getTableName()); tableConfig.setValue(HoodieTableConfig.NAME, cfg.getTableName());
tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()));
tableConfig.setValue(BASE_FILE_FORMAT, cfg.getString(BASE_FILE_FORMAT)); tableConfig.setValue(BASE_FILE_FORMAT, cfg.getString(BASE_FILE_FORMAT));
} }

4
hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java
View File

@@ -37,8 +37,8 @@ public abstract class BaseKeyGenerator extends KeyGenerator {
super(config); super(config);
this.encodePartitionPath = config.getBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.key(), this.encodePartitionPath = config.getBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.key(),
Boolean.parseBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.defaultValue())); Boolean.parseBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.defaultValue()));
this.hiveStylePartitioning = config.getBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), this.hiveStylePartitioning = config.getBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(),
Boolean.parseBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.defaultValue())); Boolean.parseBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.defaultValue()));
} }
/** /**

22
hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java
View File

@@ -36,21 +36,21 @@ public class KeyGeneratorOptions extends HoodieConfig {
.defaultValue("false") .defaultValue("false")
.withDocumentation("Should we url encode the partition path value, before creating the folder structure."); .withDocumentation("Should we url encode the partition path value, before creating the folder structure.");
public static final ConfigProperty<String> HIVE_STYLE_PARTITIONING = ConfigProperty public static final ConfigProperty<String> HIVE_STYLE_PARTITIONING_ENABLE = ConfigProperty
.key("hoodie.datasource.write.hive_style_partitioning") .key("hoodie.datasource.write.hive_style_partitioning")
.defaultValue("false") .defaultValue("false")
.withDocumentation("Flag to indicate whether to use Hive style partitioning.\n" .withDocumentation("Flag to indicate whether to use Hive style partitioning.\n"
+ "If set true, the names of partition folders follow <partition_column_name>=<partition_value> format.\n" + "If set true, the names of partition folders follow <partition_column_name>=<partition_value> format.\n"
+ "By default false (the names of partition folders are only partition values)"); + "By default false (the names of partition folders are only partition values)");
public static final ConfigProperty<String> RECORDKEY_FIELD = ConfigProperty public static final ConfigProperty<String> RECORDKEY_FIELD_NAME = ConfigProperty
.key("hoodie.datasource.write.recordkey.field") .key("hoodie.datasource.write.recordkey.field")
.defaultValue("uuid") .defaultValue("uuid")
.withDocumentation("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n" .withDocumentation("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n"
+ "Actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using\n" + "Actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using\n"
+ "the dot notation eg: `a.b.c`"); + "the dot notation eg: `a.b.c`");
public static final ConfigProperty<String> PARTITIONPATH_FIELD = ConfigProperty public static final ConfigProperty<String> PARTITIONPATH_FIELD_NAME = ConfigProperty
.key("hoodie.datasource.write.partitionpath.field") .key("hoodie.datasource.write.partitionpath.field")
.defaultValue("partitionpath") .defaultValue("partitionpath")
.withDocumentation("Partition path field. Value to be used at the partitionPath component of HoodieKey. " .withDocumentation("Partition path field. Value to be used at the partitionPath component of HoodieKey. "
@@ -67,24 +67,24 @@ public class KeyGeneratorOptions extends HoodieConfig {
@Deprecated @Deprecated
public static final String DEFAULT_URL_ENCODE_PARTITIONING_OPT_VAL = URL_ENCODE_PARTITIONING.defaultValue(); public static final String DEFAULT_URL_ENCODE_PARTITIONING_OPT_VAL = URL_ENCODE_PARTITIONING.defaultValue();
/** /**
* @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods. * @deprecated Use {@link #HIVE_STYLE_PARTITIONING_ENABLE} and its methods.
*/ */
@Deprecated @Deprecated
public static final String HIVE_STYLE_PARTITIONING_OPT_KEY = HIVE_STYLE_PARTITIONING.key(); public static final String HIVE_STYLE_PARTITIONING_OPT_KEY = HIVE_STYLE_PARTITIONING_ENABLE.key();
/** /**
* @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods. * @deprecated Use {@link #HIVE_STYLE_PARTITIONING_ENABLE} and its methods.
*/ */
@Deprecated @Deprecated
public static final String DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue(); public static final String DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING_ENABLE.defaultValue();
/** /**
* @deprecated Use {@link #RECORDKEY_FIELD} and its methods. * @deprecated Use {@link #RECORDKEY_FIELD_NAME} and its methods.
*/ */
@Deprecated @Deprecated
public static final String RECORDKEY_FIELD_OPT_KEY = RECORDKEY_FIELD.key(); public static final String RECORDKEY_FIELD_OPT_KEY = RECORDKEY_FIELD_NAME.key();
/** /**
* @deprecated Use {@link #PARTITIONPATH_FIELD} and its methods. * @deprecated Use {@link #PARTITIONPATH_FIELD_NAME} and its methods.
*/ */
@Deprecated @Deprecated
public static final String PARTITIONPATH_FIELD_OPT_KEY = PARTITIONPATH_FIELD.key(); public static final String PARTITIONPATH_FIELD_OPT_KEY = PARTITIONPATH_FIELD_NAME.key();
} }

6
hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
View File

@@ -280,7 +280,7 @@ public class FlinkOptions extends HoodieConfig {
+ "By default true (in favor of streaming progressing over data integrity)"); + "By default true (in favor of streaming progressing over data integrity)");
public static final ConfigOption<String> RECORD_KEY_FIELD = ConfigOptions public static final ConfigOption<String> RECORD_KEY_FIELD = ConfigOptions
.key(KeyGeneratorOptions.RECORDKEY_FIELD.key()) .key(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.stringType() .stringType()
.defaultValue("uuid") .defaultValue("uuid")
.withDescription("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n" .withDescription("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n"
@@ -288,7 +288,7 @@ public class FlinkOptions extends HoodieConfig {
+ "the dot notation eg: `a.b.c`"); + "the dot notation eg: `a.b.c`");
public static final ConfigOption<String> PARTITION_PATH_FIELD = ConfigOptions public static final ConfigOption<String> PARTITION_PATH_FIELD = ConfigOptions
.key(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) .key(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
.stringType() .stringType()
.defaultValue("") .defaultValue("")
.withDescription("Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`.\n" .withDescription("Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`.\n"
@@ -301,7 +301,7 @@ public class FlinkOptions extends HoodieConfig {
.withDescription("Whether to encode the partition path url, default false"); .withDescription("Whether to encode the partition path url, default false");
public static final ConfigOption<Boolean> HIVE_STYLE_PARTITIONING = ConfigOptions public static final ConfigOption<Boolean> HIVE_STYLE_PARTITIONING = ConfigOptions
.key(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key()) .key(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key())
.booleanType() .booleanType()
.defaultValue(false) .defaultValue(false)
.withDescription("Whether to use Hive style partitioning.\n" .withDescription("Whether to use Hive style partitioning.\n"

18
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala
View File

@@ -65,7 +65,7 @@ object DataSourceReadOptions {
.noDefaultValue() .noDefaultValue()
.withDocumentation("Comma separated list of file paths to read within a Hudi table.") .withDocumentation("Comma separated list of file paths to read within a Hudi table.")
val READ_PRE_COMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD val READ_PRE_COMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD_NAME
val ENABLE_HOODIE_FILE_INDEX: ConfigProperty[Boolean] = ConfigProperty val ENABLE_HOODIE_FILE_INDEX: ConfigProperty[Boolean] = ConfigProperty
.key("hoodie.file.index.enable") .key("hoodie.file.index.enable")
@@ -238,7 +238,7 @@ object DataSourceWriteOptions {
* key value, we will pick the one with the largest value for the precombine field, * key value, we will pick the one with the largest value for the precombine field,
* determined by Object.compareTo(..) * determined by Object.compareTo(..)
*/ */
val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD_NAME
/** /**
* Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. * Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting.
@@ -252,20 +252,20 @@ object DataSourceWriteOptions {
* the dot notation eg: `a.b.c` * the dot notation eg: `a.b.c`
* *
*/ */
val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD_NAME
/** /**
* Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`. Actual * Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`. Actual
* value obtained by invoking .toString() * value obtained by invoking .toString()
*/ */
val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME
/** /**
* Flag to indicate whether to use Hive style partitioning. * Flag to indicate whether to use Hive style partitioning.
* If set true, the names of partition folders follow <partition_column_name>=<partition_value> format. * If set true, the names of partition folders follow <partition_column_name>=<partition_value> format.
* By default false (the names of partition folders are only partition values) * By default false (the names of partition folders are only partition values)
*/ */
val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE
val KEYGENERATOR_CLASS_NAME = ConfigProperty.key("hoodie.datasource.write.keygenerator.class") val KEYGENERATOR_CLASS_NAME = ConfigProperty.key("hoodie.datasource.write.keygenerator.class")
.defaultValue(classOf[SimpleKeyGenerator].getName) .defaultValue(classOf[SimpleKeyGenerator].getName)
@@ -565,7 +565,7 @@ object DataSourceWriteOptions {
val DEFAULT_ENABLE_ROW_WRITER_OPT_VAL = ENABLE_ROW_WRITER.defaultValue() val DEFAULT_ENABLE_ROW_WRITER_OPT_VAL = ENABLE_ROW_WRITER.defaultValue()
/** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */ /** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */
@Deprecated @Deprecated
val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key() val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key()
/** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */ /** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */
@Deprecated @Deprecated
val DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue() val DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue()
@@ -604,13 +604,13 @@ object DataSourceWriteOptions {
/** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */ /** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */
@Deprecated @Deprecated
val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD.key() val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()
/** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */ /** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */
@Deprecated @Deprecated
val DEFAULT_RECORDKEY_FIELD_OPT_VAL = RECORDKEY_FIELD.defaultValue() val DEFAULT_RECORDKEY_FIELD_OPT_VAL = RECORDKEY_FIELD.defaultValue()
/** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */ /** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */
@Deprecated @Deprecated
val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD.key() val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()
/** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */ /** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */
@Deprecated @Deprecated
val DEFAULT_PARTITIONPATH_FIELD_OPT_VAL = PARTITIONPATH_FIELD.defaultValue() val DEFAULT_PARTITIONPATH_FIELD_OPT_VAL = PARTITIONPATH_FIELD.defaultValue()
@@ -620,7 +620,7 @@ object DataSourceWriteOptions {
val TABLE_NAME_OPT_KEY = TABLE_NAME.key() val TABLE_NAME_OPT_KEY = TABLE_NAME.key()
/** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */ /** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */
@Deprecated @Deprecated
val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD.key() val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD_NAME.key()
/** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */ /** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */
@Deprecated @Deprecated
val DEFAULT_PRECOMBINE_FIELD_OPT_VAL = PRECOMBINE_FIELD.defaultValue() val DEFAULT_PRECOMBINE_FIELD_OPT_VAL = PRECOMBINE_FIELD.defaultValue()